{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformer的Keras实现\n",
    "\n",
    "参考tensorflow的官方教程：[transformer](https://www.tensorflow.org/alpha/tutorials/sequences/transformer)\n",
    "\n",
    "* tensorflow==2.0.0a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0-alpha0'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!pip install -q tensorflow==2.0.0a\n",
    "!pip install -q matplotlib\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## positional encoding\n",
    "\n",
    "\n",
    "$$PE_{pos, 2i} = sin(\\frac{pos}{10000^{2i/d}})$$\n",
    "\n",
    "$$PE_{pos, 2i+1} = cos(\\frac{pos}{10000^{2i/d}})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_angles(pos, i, d_model):\n",
    "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
    "  return pos * angle_rates\n",
    "\n",
    "def positional_encoding(position, d_model):\n",
    "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
    "                          np.arange(d_model)[np.newaxis, :],\n",
    "                          d_model)\n",
    "  \n",
    "  # apply sin to even indices in the array; 2i\n",
    "  sines = np.sin(angle_rads[:, 0::2])\n",
    "  \n",
    "  # apply cos to odd indices in the array; 2i+1\n",
    "  cosines = np.cos(angle_rads[:, 1::2])\n",
    "  \n",
    "  pos_encoding = np.concatenate([sines, cosines], axis=-1)\n",
    "  \n",
    "  pos_encoding = pos_encoding[np.newaxis, ...]\n",
    "    \n",
    "  return tf.cast(pos_encoding, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 50, 512)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXl4VNX5xz/n3lmTmewrSSCsAoosooJYFfd9t6K1xarVWqu1LnVrtVVrtbbazbqW/tSquFVFxAVF6wqyiMoiENaQhOzrZNZ7z++PeyeZhAADJEjwfJ7nPHebO3MyDGfOvO/5fl8hpUShUCgU3w20b7sDCoVCodhzqEFfoVAovkOoQV+hUCi+Q6hBX6FQKL5DqEFfoVAovkOoQV+hUCi+Q/TpoC+E2CCE+FoIsVQIscg+lyWEmCuEWGNvM/uyDwqFQvFtIYSYIYSoEUIs28Z1IYT4mxCiTAjxlRBiQsK16fY4uUYIMb23+rQnZvpTpZTjpJQT7eObgfeklMOB9+xjhUKh2Bf5P+DE7Vw/CRhut8uBh8GaHAN3AIcChwB39NYE+dsI75wBPGnvPwmc+S30QaFQKPocKeWHQMN2HnIG8JS0mA9kCCEKgROAuVLKBillIzCX7X95JI2jN55kO0jgHSGEBB6VUj4G5Espq+zrW4D8nm4UQlyO9c0HwnFQjtSoT0mjZGAhrnVlrNVTGOmM4C3M5YtNzYwr9NCwsY7WksG0NLZyYIGLTasqSHdouEaNZNW6SlypfkYXptC8fA2tMZPcbC+OgUMoqwnQ3tQEpoHD6yMrK5UBfjc0VRPY0kRryCAqJQ4BKbqGx+9C97hwpqeBx0/YFLRGDNpCUUJhg1jUwIxFMGNRpGlab0Nc+SwECA2haQihIXQdoelomo4QAqFhbwWaJtCEQNcFuhBoGvbWOq8J6yk1Iaynje/HXwbrPFjX7Pe18z3u8n53e/+3+gfZwfUdnN/lR27jYS3hGOlOgRQaWqSdNa2QWr6BgvH7s3JzM+m15eQduD8r1lUxOjVKa22A0OCh1FTVMm5EEVVLl2NIKB5ZzKoWB+2N9fhzcxieptH0zXqaYyaZHgf+wQNo1lKoqGsnFg5hhINoDhduv4+8dA+ZHici0EC4oYlwc5j2mElUSiTWjMohBC5N4HJpOL1OHCluNI8bzZ2CdLiQmgNTQsyURExJ1DCJGCbRmCRimBiGiTQlpimRJkgp7WaCaSLtz5aU9mdMmkjo/LzZ2y7n2IEKv5+r9GWwvk5Kmbur92tpxZJYKNnXWg4kPvgxe5xLliKgPOF4s31uW+d3m74e9A+XUlYIIfKAuUKIbxIvSiml/YWwFfYb9xiAlpIjzwn6+L/RJ3LrP25h4Pmnc0bmQTxVuJEDb7sC38/f4uObh/PclU/y3u+eYu7LH/DZDSVcc8TNnJCdysA33ueIC37HwIOn8tltE3jjgBN4v7adK08bQ97fZnL6Q/P54rVXiYXayBs9hQunTeL2Y4bAK/ez8E9v8L9v6tkSipHl1JmQ4WG/oweROaKIvBNPRO4/lbXtDj7a2Mj/VtWwZn0jDVWttFZvJNRYTTTYhhmLIE0DAM3hQnO4cHp9ODypuFLTcaam40pJxe1x4vI6cLh03B4nbq+DFI+DjBQnPo8Tv9uBz2M1r1MnxamjCYHboeFxaDg1a9+paTh10bHVhUC3f9Pp9heEJhL2sb4M4l8i8XPQ+SWhia7jb+dju47KWpJfDlr3b5ltsK2HzV3XxAnFLqIOL95NizjtA51DfvkjbvrkEybc8g6nP3QtP3vvQ8aefy9vTKrig0c+ZcVfXuBvf3icT9++k7uzx9EcNfnTjD9y5PsZLH7xGaZccRmzj3cxe8rFzNnSxjml2Ux9+k7meA/i1hmLqFm7mqYNy0jNLWHE4Yfzs1NGct7oXPTPXmDDzFcpe7OMpfVBKkNRDAkuTZDj0hmc6qS4JI38MXnkHDgE/8gRuIYdiJlVQtiXT3vUpC5oUNkapqIlxOamIJsbg1Q1BWlqDRMKRAkHo0SCMSLhGKZhEg21Y4SDmLEIRixiTTKiEfuzZiJNA2kamPbnThpGx2cwvu2+v71z/Yno0n9v3K0niIVw7Hd6sq8VSghd9wv6NLwjpaywtzXAK1ixqWr75wv2tqYv+6BQKBQ7hRAITU+q9QIVQEnCcbF9blvnd5s+G/SFEKlCCH98HzgeWAbMAuKZ6OnAa33VB4VCodh5RMcv8h21XmAW8CN7Fc8koNkOf78NHC+EyLQTuMfb53abvgzv5AOv2D//HcCzUsq3hBALgReEEJcCG4Hv92EfFAqFYuewZ/q981TiOeAoIEcIsRlrRY4TQEr5CDAHOBkoA9qBH9vXGoQQdwEL7ae6U0q5vYRw0vTZoC+lXAeM7eF8PXDMzjxXanY2l40p5o3sSfxw0/O4P3iSQfev55lHr6P2gSMZONnB+zf8luOumMztb37B6CMOZuXf76PA42DMufvz5wUbiQaaGT++EHPRHL5uDpPvdlB0xDi+rA1SU95MLNSG5nCRnp/HmKJ03K1bqF5dTlNVG20x0+qHruFPd5OSl0ZqQTaO7AICDi9NoXYa2yPUt0WIBGMd8dZ4XLV7jNRK3mpoTlfnT0Uh0BwaukOzkrEaCE3gcmjommbH5TtbPCauC6tZid3O+H18mxgT77K/jfe6pxh69zh99+NtnU8+qZt8X+IM/M10Dij4KQ9/cA8PX/8PZp2WRm3jCZz08AKe+uX3eOkhuPjpJYw75Tjeu/unHHXZIdw5ZxUDxh6BOfcJasMGEzI8xMadQvk/nsGTnssZ44sILniaFS1hfA6NvDF5yIFjWLykiZa6RkKN1QB4MwvIyEmhJN2DI1BHtHoT7TVtNIdiBAwTw85S6QK8uobPoeFOc+NK8+JM9aKl+BEuL9KVQsSQdjNpjxqEYibBiEEkZhKJmRgxK5lrGhLTTtiaZmcarOMzZmz9Oet4jNG/Y/R7GoH1f7Q3kFJesIPrErhqG9dmADN6pSMJ9HUiV6FQKPoXQqD10kx/b0QN+gqFQtGN3grv7I2oQV+hUCgS6cWY/t6IGvQVCoUiAYFAczi/7W70Gf3CZXOEX5L11Ku8ff/ZPDj9cS791OTZm44ix+Xg2oc+4zeXHsycihYKr/sddasX8pvT9+fTd9YzpTSdQdMv4sNPN+FMTWfaxBIq3pxHdTjG6DQXqYcezScbG2iuXA+AKzWdzHwfo3N9iC1raCqrpDZsEDRMXJog3amRku0lpSAbd14OZmoWbRGThmCMmpYwoWCUSDjWKZqJRrok1+JJWy1hnW986Zfu0NA0W4lrJ3R1TeCwE7cuh2Ynde1kbjx5m5jU3UaGtXsyd1uJ2DjdhVm9TbLCrO3xyH9XsXnRu7yyspbZDz3Bu0dcSN3F97Bg5guM/uQhLjhtOEtmvcWjPxjP/IYgxb+4lYol73PyscNY/ths0p0aEyYX8e76Jho3LiO9ZBRHD85i8/tLqA7HyHc7KJg4jEZXNks2NhKo2UQk0Izu8uLNzGN4vp+iNDd6aw2BilraqgM0R00iCUlWlybw6gKPx4Er1YnLn4ozLQUtNQ3T5UU63ERsJW48iRuKWUncYCRGJGZaKlxbkWvGLHVuYuLW7GGhQHdhlmIn2bPr9Pc4aqavUCgU3eivA3oyqEFfoVAoEhGi15Zs7o2oQV+hUCgSEOzbM/1+EdPf8s0mvnf1c2i//hFRKXnxn08z/K37mX7jUaz/eBYXZdWS5dJ5cr3ElZrOUSl1LGsJMfbSw2kccQyVyxaRPWwCU0vTWf/uWgwJJRMKiA06iPdX1hCsr0R3eUnJHsB+gzIoSXMS3fgNzRtbqA0bHeZZWS4dX34qqQXZ6NmFmCmZtEVM6tsjNAQihIMxouEYRiRoO2z2IMxKiONrHTF+ga5raLq91QRCiI4YvsuhdcT2rXi+FcuPx/UhLtDqFGl1NFsiZZmodY2lJ5qt7Qp9FfNPhnsfvZAH/nojN954JIXjj+XVdY2cffc8UrIHMPOq/7D/Q/8k3NrAoCUzGeBx8HpDGkYkyC++V8r8TzYzKcvLqB9OZcanG4gGminar5hS0Uj5J+UEDckwn5P0ceMoawxRWd5MpK0RMxbBlZqOP8vL8AIfOV4HZs0m2ipqaa8P0hw1OmL6icIsZ6oLd7obZ1oKeqofLdWPdKZgOtwd4qxQzCRsC7PaIwbhBGGWETMxY6YV14/H9Lt9tjrN1Mwe369kzdYUgNDQHa6kWn9EzfQVCoUiEbFvz/TVoK9QKBQJCNQ6fYVCofhOsS8P+v0ipu/QoKl8JX+fsZTrH70Ity+Tx697Ccd1fyGteARfXHUjZxxdyp+f+5LSSVOpevhPVgx+2uU8t6yaQG05g8eU4F3zEV9taibLpVNy1GjKWiQV6xqJBJrxpOfgyy9h/KAMMmSA1tVradncQkvMinv6HBrpHgcpeT6cufk4cgowvBm0hA3q2yPUt4UJB6NEQyFikWCXwilxhKZ3mK0J3Y7tO11outZRKUtowortd8Tz9R7N1hLj+l0M2BLM1uL0ZITWfa28Jrqv5+8snhK/p6fn2ll2t3hKnGt953Lam7/nq+n3Me/ek/nREQPZ+OnrXHf9eSxsDPG7LyIMPvxkPr7+cU6eOoh7Xv6a7GETGFjxGStbwxxw1ihcx/6I5V9Uobu8HHtQEeaX77GmohWXJhgwJg/H6EksqWqhobqNSKAZAHd6Dhm5qQzNSsVvthOrWm9VV2sOE7LX3IOVA/JowjZbc+Hye3D5UxApaQivH+l0E46ZHTH99qhJOGZYZmuGZbZmGlYsX8oEszX7c9WlGVvH63cVFedHrdNXKBSK7xYqvKNQKBTfGYQQaM7+uTInGdSgr1AoFIkowzWFQqH4brEvD/r9IpGbs/9w7rv/Gk4uSuPVAy7jjt9cRGUoyjkPL2DaxSfz4rvrGf/Ab9nw6dv8/JwDWPD4fI7ISWGZKOI/75ahu7z88HuDqXn9FcqDUUb4XGR+7yg+3tRIY0Ul0jRIzR1IdoGfMXl+HHXraFxdTnVrhKAh0QWkOTRS81NJLcxGzy4Afw6tYYO69gi1LWFaA1bVLCMcxIxGtjLC6m62pjk6q2bp8YpZDq1DnKVrAneiwVqC8VpipSyw9uOirUREQnI2Lsza3UTstujtqlk74tn7/8Hdd85l+rUPE7j6fCa8+SZDjzqTmwsrOWdkNk888Q73/eQQ3lhVx7jf38iajz5k3NSxrH3oEXQhGHTR91ka9FO7ajHpxSM464BCquZ+wIb2CDkuncKJpQSzhvDpmjoCtZuQpoHmcJGSXURpvp9BGR70liraN1fSWtlGQ8ToqLDm0oRttqbhdem409240lJxpaWi+TOQLi/SmZKQxDUIxwxChiXOiputGTFpi7OkbbTW1WwtkUTxVaLZmqqatWto9sKKHbX+SL8Y9BUKhWJPIYS1ii6ZluTznSiEWCWEKBNC3NzD9QeFEEvttloI0ZRwzUi4Nqs3/j4V3lEoFIpu6HrvzIeFEDrwEHAcsBlYKISYJaVcEX+MlPKXCY+/Ghif8BRBKeW4XumMjZrpKxQKRSKC3pzpHwKUSSnXSSkjwEzgjO08/gLguV74K7ZJvxj0V1SHuGDxPzl+yWyu/fWT/DT8MZecN4olr7zMA0fnETEl74r9APjxyFQ+rGtn3EUT+PP7ZWxY8iWZpQdwyogcyl7/kqAhGT4qB0ZO4Z3lW2ir3oDmcJFRmE/pwHSGZnqIlH1FQ1k9W0IxIqbEq2uW2VpeCr6iXBy5RRip2bRFLbO1mtYw4WDMKqBiC7PM6DbEWQmxfat4isP+AFmxeaGBpnctmNIltp8oyrJj+3o8br8ds7XEbZye/vGT/UAkmq19G6HN066+gh8fOxjd7eWhmSs48k+f8uotR/HWCddw9It/pGHdl5xiLselCb7IPpT2+kruOmU0n/93JRMyPLSPPZXH52+kvb6SotH7cUAGbPpgDc1Rk2E+F7mTx7O2MczaDU2EGqsBbLM1H/sXpZGf4kDWbKK1vIZATdcCKrqw4vo+h8Cd5rZahg891YeW4sd0pmA6PXZM3zJai5uthWOWMCsSMTAMsyOWb9iGa/F4fmIBlW2ZrW0vnq9EWNvGctnstUG/CChPON5sn9v6dYUYBAwG5iWc9gghFgkh5gshztzFP6kLKryjUCgUXRA7U90tRwixKOH4MSnlY7v4wtOAl6SUid/Ig6SUFUKIIcA8IcTXUsq1u/j8gBr0FQqFoit2eCdJ6qSUE7dzvQIoSTguts/1xDTgqsQTUsoKe7tOCPEBVrx/twb9fhHeUSgUij1JL4Z3FgLDhRCDhRAurIF9q1U4QoiRQCbwWcK5TCGE297PAaYAK7rfu7P0i5l+uLWJu699ic/bTiLa3sJ/zr2HCyuX4j71bsp+8RPOOqiQXzy1mIGHHEfT43cBMOjKq/n0/o20bF7NxPMuIL9mKa+uaiDdqTHomJFsjKaytqyeUHMt3sx8cor8HDo0m1xHhNbVq2je2EKLve7a59DIcTvwDfDjLijATM3GTMmkpTFKrW22FglGiYYjttladJtma5rDaZmsJZit6XaLF0SPr9PXNYFL7yyk0t1sLb4+34rhW6+zLbO1jrg+du6g47zYeo39Xm62BvCk+202PvUas8JRAmUvMuOV50g1XuD1zS1sahtKyaGnMP+nv+XUgwq57vmlZJQewPjIap5sDHH5tNH895s6PvpsE5rDxeETihBfvcM3qxvQBQzaLxvXgUewYHMz9VtaiQSacXh8ttlaCsOyU0nXokSrNtC2uY62xhABozOm79U1PJpVQMXlc+JOc+NKS0HzZ6KlphFzea3CKfYa/XgLRgyCUauIStxszTCsJqXc2mgtSbO1ngqobO9x33WEAN3RO4kqKWVMCPFz4G1AB2ZIKZcLIe4EFkkp418A04CZUkqZcPso4FEhhIk1Qb83cdXPrtIvBn2FQqHYk/RmVTgp5RxgTrdzt3c7/m0P930KjOm1jtioQV+hUCgSEKL/qm2TQQ36CoVC0Y2dSOT2O9Sgr1AoFN3Ylwf9frF6p7AonyNyUlj4/H+44bZL+LI5zEkPL+DMS87muRdWcNgjt7Nq3pv8bNqBzP/ze0zJ9rIyZSRbln2M0HQuOmoIta/OZHVbmBE+F3nHHsP/NjRQu35zh9nahCHZjCtMw1lbRuPKjWxpCtEWMxPM1ixhlm4Ls1pjguq2CFuaQjS3RQgHY8SCbRjhIEa3qlndzda6irREF7M1XddwODTcDs2qmtXNcC3RbE1PSOZ2T+DurNlaL4Yw+9xsDeCmH87giEv/TvY9P+HI+W8zcPKpPHrfPM4YlM5dD77J76+cxEvzNzPpLzeybO4HHHjMIax78E8ADP/Jhcx4by1bli8mrXgEF0woovrNt1kbiJDrdlA0ZQjBvP34eE0tLVUbMGMR3P5My2yt0M+w7BT05graN2ygtcoyWwsaVtJfF3RUzPK5HXgyPbgz/Lgz/GipfqTTMluLV81qj5q0R5M3W4OtE67dzdZ2BZXETUD0IHTcRuuPqJm+QqFQJCAQaI5+MR/eJdSgr1AoFIkIVCJXoVAovkv05pLNvY1+8RsmN1THKcveZr/jzuEm8SmXTxvNgpkv8NiJBbTFTOZ6xyNNgyv39/FuTYBDf3ww9763mmigmawhYzlrVC6rXl5M0JCMGpMHY45m9ldVHWZrmUUDOKQ0kxFZXiJrllK3qnYrszV/oa/DbC2ke2kOGx1ma6H2KOFgFCMSxIiEdmi2pjtcHWZrmkPrYrYmEoRY3c3WXPECK7tgttZ94rIjs7Xtx/+/XbM1gOnHD0Fzunjw8SVM+csS3vztcehCcPycv1K3eiHnYpmtLS08kkBtOX866wA+fu5rJmR4CE48i3VLviFQW07JAaMZnwlr31pBc9RklN9F/pSDKGsMs3pdY4fZmjezgLScdA4sySA/xQHVG2gtr6Gtqo2GiEnQsDQ18eIpPZqt+TIw3T5Mp4eQbbZmFVCx4vntEWO7Zmvxz9WOzNbMHYi2VPx++1iGa8m1/kifd1sIoQshvhBCzLaPBwshFtgFBZ63pckKhUKxdyBU5azd5RfAyoTj+4AHpZTDgEbg0j3QB4VCoUgSgaZrSbX+SJ/2WghRDJwCPGEfC+Bo4CX7IU8CveIRrVAoFL2B2Mdn+n2dyP0L8CvAbx9nA01Syph9vL2CApcDlwP40Dn0wa9Z8LtjeCR/LBdVfEHK+Q+y/JKLmTa1lEufWMiQKSdS+9ffoAso+dl1fHTXN6TmljB04khyy+fz/Mp6slw6g08cQ1nIw9rVltlaSvYA8gemM67AT57WTtOy5TSta6IxasU9fQ6N3BQn/uJ03AUFGL5cmsMGzSGDLW1halpChAIRIsEg0VAb5jbW6CdrthaP57sc+o7N1jrWE4Ou9a7ZWsdxt+faVXrTbA2g5e/P83G6m5qaV5nx6kxE7RNccfsJ3Fs1gCFHnMGHP/w15xxdytVPLSZ72ATGNC7m8cYgV18yjueW1dC4YRm6y8vxkwbCotmsLGtEFzDwgFxc46fyWXkTdZUthFsbcHh8+PMKycpPZb9cH+kiTLR8Na2bamlu2NpszefQSHfquNNceDK8uDN8ltmaL4OYy9uxRr813Gm21haK9YnZWhwVx985lDhrFxBCnArUSCkX78r9UsrHpJQTpZQTvei93DuFQqHoGSHYWhS5jdYf6cuZ/hTgdCHEyYAHSAP+CmQIIRz2bH97BQUUCoXiW6G/DujJ0GczfSnlLVLKYillKZZX9Dwp5Q+A94Fz7YdNB17rqz4oFArFziJIbpbfX78Yvg1x1k3ATCHE3cAXwL++hT4oFApFjwgBrn3YhmGP/GVSyg+klKfa++uklIdIKYdJKc+TUoZ3dH9mipPlc15k0VFHUxmKcuy9/+O668/jqdlrmPjEXyj732zuuPgg5v39Q44t9POpUUz11x9SPO4Qrjx2OJUzn2VtIMIBaW5yjz+ZuWvrqFu/Hmka+PIHM3l4DoPSXehbVlG/fD2VLeEOs7VMp45/gC3Myh+ImZpNa9ikJhBmS1OI1kCEiG22ZkYjGNHtm61ptjDLEmdpW5mtuWyzte4zCpdD267ZWiI9ma1tDyF674Owp+Y+Z/z4HurPO5Xhb73DmFO/z0OPLKTh4j/wwJ9fZMYvD+flZTVMfOheVrw7l6mnHcqK3/8ZlyYYdtVPmfH2aoxIkMzSA7hoQjGbX5vD2kCEAR4nA48cSXPGUN5dUU1L1TqkaeBJzyEz38d+JRkMy0rB0VhO2/pNtJS30hAxaLMrrLk0gUcTpDs1vC4db6YHd6Yfd6YfzZ+BdFlmayFDEo51Vs0KxKtmRWIEI0aPZmvxBQLdTde6m62ZCZ+9ZJO3KsnbFSHAoYmkWn9E2TAoFApFAoJ9O6avBn2FQqFIRPTfeH0y7LuBK4VCodgFrJm+llRL6vmEOFEIscq2nrm5h+sXCyFqhRBL7XZZwrXpQog1dpveG39fvxj0XcNHcOwVl/Hs55Vcf89pLJ/zIjcXVpLp1PnrJh+u1HTOSavhk/ogk246gdtnLceMRTj7mKGcNSqHFS98QcSU7DepiNjoo5m1uIK26g04PD5yBuYzqTQLV/UqwssXUPdNPVtCBoa0hVlunbRiP/6B+Wh5AwngojoQpiZgma0FWyOEQ51ma90LWYjusfzE4im6hqZbW90hcCSaq3UUUtE64vbdzdbAEk0lY7YWn7fE4/fbcxHsbbO1vig2UXLQUTzz0SYmXT+bT2+azCi/m7PvmUd7fSXjFv+bEq+TmS0DCLc28MdTRzF3dhnH5PkoL5nC+kVL8BcOZciEkYzQ6il7czVtMZOxGR5yvjeFr2vaWbe2gfb6SoSmk5o7kKIBfg4sSacg1YFRWUbLhqqOAipxYZbLLp6S5rTi+VYBFR+6PwPdn4Hp8mE4PIRjklBsa7O19oiBERdlxWyBVszEiMWQhrFV3L5TqGV2eW/ioq2O412I83/X6a3VO0IIHXgIOAkYDVwghBjdw0Ofl1KOs1vcwSALuAM4FDgEuEMIkbm7f1u/GPQVCoViT6EJa9KVTEuCQ4AyewFLBJgJnJFkV04A5kopG6SUjcBc4MRd+qMSUIO+QqFQdEO3LU921IAcIcSihHZ5t6cqAsoTjrdlPXOOEOIrIcRLQoiSnbx3p1CJXIVCoUggbsOQJHVSyom7+ZKvA89JKcNCiCuwjCiP3s3n3Cb9Yqb/zcY6Zk1u45IThrD41FspOfQU3jrhGn50zRT+/M/3GH/aSSz71S0UeBz4Lv41Kz/6gszSA7j04GLEh8+woLyFEq+T4WdPZmFVO5tW1RFubSAlZwBDhmYxJi+V2JolNHy1irr1nWZraQ6d7EwP/uJMXEWDMH05NIUNtrSGqWoJUdUUJNQeJdIeIBrcvtma0LStzNbiJmu6w7JpjRdNcTl02zytM77fk9laYkH0+LZ70ZTEcHr32Lomul7fXbO13Y3c70zof9lNI/n170+h+usP+eSw47h49p2s/3gWh077Pi9c/i+m/fww7nhiIQMnnUz2xzNY3RbhoKuP4C8fbaBl82qKDxzPD48aQmTeM3xR0YrPoVFyeDHamKP437p66ivqiAaacaWmk56fw4RBmYzO9eELNxDdsJKWjQ00tIRpiVlma7oAr26t0Xenu/BkevBkpuLJ8KP5MhAp6Uh3KqGYScgwaYtYBmtt4ViH2VowYhCLGpgxE9OQmFJuZbZmJpitqfh839GLitwKoCTheCvrGSllfYJe6QngoGTv3RX6xaCvUCgUe4peFmctBIbbxaNcWJY0s7q+nihMODydzvojbwPHCyEy7QTu8fa53UKFdxQKhSIBgeg1GwYpZUwI8XOswVoHZkgplwsh7gQWSSlnAdcIIU4HYkADcLF9b4MQ4i6sLw6AO6WUDbvbJzXoKxQKRQI7GdPfIVLKOcCcbuduT9i/BbhlG/fOAGb0WmdQg75CoVB0YV+3YegXMX1pxPjb4T9n8AuzmX7LM8y64zhe39xC6m0PU/vNfP49/SBee30NJx85kMe/bqBh3Zfsd9hYBpR/yppWHz0lAAAgAElEQVR/v0xlKMZBhT5Sjz6Hl76spGH9CoSmk1Eygqmj8ijU22leupTaLzeyqT1G0DBxaaJDmJVWWohzQCmGL5fGoFUxa3NDkEBrhHAwSizYZomzejJb03V0W5iVKNKyErgJAq2Otb/6VmuBdVuU5dQETq3TbE3rSNp2CrOg03CtQ6TF9gVSiR+CjgRwD4/bnqBrT/PgiNN44Yjr+dWdV/PC1zXc2z6WIUecwVs/PYT5DUFy7niETfPncMOPJvDJLU9T4nWSc+mNzHm3DN3l5bQjB3PWyBxWP/8h5cEoQ1NdDDp2PJtFJvOWbaG1sgwAb/YAsgt9jClMY3CGB0fDRprXVtC0sZnasEHQ6DRbS9WtilmeDI9ltpbhx52Vjp6ejelOxXSlEox1NVtrC8Vot83WwhED05DEokYXcVaPVbM6BFpm0mZryZ77zqOKqCgUCsV3h7if/r6KGvQVCoWiG2rQVygUiu8I2j5eRKVfDPrDS/MJlrVx+G3v0LZlA+mPXM8Zg9I5+9EFFIydSv7cv1IZijH+rmv58XPLcKamc/1JI9n46LUsfmc9Xl0w4vRRVPiH8snSTwjUluP2Z1EwKJPJxZloGz+n5osy6lbVUxeJYUjIcmkUeBykF6eROrAImTmAxrBJlR3Pr2oOEmwLEw5GiYbaMGPRLuKsrYqnOF1WbN9pxfMdTt2K58cFWrYwS9cELr1rIRWnpuHUtQ5hVmLxlK0EV4guwqzECUui2Vr3iczOxut722xtZ9MFuW6dK6/7M3XXFrLm7P048g9P8vnMm1lzyTmcOSSTi579kpTsAVwyKMYtq+uZdtxg3qzzUPnlh+SMOJjpBxWTteET3v64HEPCqGGZpB11Cq9vambLhiaCjdXoLi/+/EGMKc1iv5wUClM0IouW07y2gtYtAZqjW5utpXodHWZrnuw0tPRsNH8GMbefKBphI0Z71KA10lk8pS1sxfXjRmuGYSJNaW87hViJwizYRox+O2ZriiTp5dU7exv9YtBXKBSKPYVg62p0+xJq0FcoFIpu9IUd+N6CGvQVCoUiAYFVs2JfpV9kK8Smtdww57es/3gWl9xwGf+8dx7Hz/kri//7CrddeQTv/PI5js1LZfmAI9jw+TwGHjyVEwtMvnr+a75sDjE23UPxuWfy5pp6qlZvxIxFSCsawWGj89gv201o2XxqV9RRUdNOc7SzIHpakZ+0wQU4BgzG8OfTGDKoaAmxpTlIfXOIUCBKNNCMEQ5ibMNszVqX7+xSEN3h1HssiN5lXb7W6endvSC6LjqLp3Q3W+upILomRI8x821NZhJP7ymztZ1lWvkiBk06njunzyDr8ZcRmkbqQ9cz48WVHPvSH/hg5mwmn30C626/kYgpOfC2K7hv1gqigWZGTx7OkMAaKmc+x7KWMAM8DoYcN5JA8QTeXFZFw6a1mLEInvQcsgr9TBiUQZHPibN+HYGyNTSua2JLKEZLzMSQiWv0NTyZHlJyUvBkp+PJTkdLz0Z605BuH8GoSSgmaYsYltlaKEZrONZRED0WMe01+rIjrm/GIlsZ+UFyBdF3FM9X8f5tILDyZ0m0/oia6SsUCkUCAnAmWQqxP6IGfYVCoUhgXw/vqEFfoVAoEhH9N3STDGrQVygUigR25FXV3+kXgava5jCXbd6P7/34x/yltJxUXePeqgE4vT5+klPN29UBjr3rDH4xcymxYBsXnTqS9pf+wSf1QYKGZNxRA4lNOJ2Zn22kqXwlDo+P/CFFTB2eg7dmFdWfr6Bqcyub2qNETInPoVHkdZAxKI30oUXoBYMJCA+VrWEqm4JUN4UItkYIh6LEQm0YkRBmD2ZrurMziRs3XXO4nJ0ma7pluuZINFuzhVmdSVxr1qELuiZ07eRtotlah8FaQvWsLklZthZh9WS21hOJ920l7NrGPX35H2f4FS/y1W8PZZTfzdRb3+a231zMo/fNY4DHyVPGaELNdcy4YCyznl3GSSVpbBpxEt98+Bn+wqFcd8xw6l78NyteWEpz1OSgnBQKTj6BhZVtrPimlkBtOULT8eUPZsigDMbmp+Fp2oSxaSVNa8pp2dxKQ6Sr2ZrPoZHpcthJXD+e7DQcGVno/gxMlw/D4SEYkwQiBq3hGK0Ru2JWxKA9YhBJEGfFjdaMWKxDmCXNrhWzrGZ2eU+6C7O6XFNJ250i/v9tR60/omb6CoVCkYAQ4NT7xXx4l1CDvkKhUCSwr4d31KCvUCgU3eivoZtk6Be/YQryfbzw4KO8c1YGM469nqsfuoAH/vwip1x8Fp9Nv54RPhfGBb/m67kfkjd6ClcdWsySf7xLW8xkhM/FiAuP4931TaxfVkU00IyvoJQxo/IYX+gjsuwTtiyuYH0gSmPUintmOnWyc1NJH5yHq3gIRnoB9UGDqtYwG+vbCbSEaW+LEG5tIRpsIxYJYsYiHf2NC7M6xFlOu3CK29vVZM2hodnCrHgc391NoKUJy3DNoWt2LB+cutgqtp8Yx9foKsaKG63F0QTdrvf8Cd9TCxh2ZVLVVr2e54dP5aIvX2Lzwre4xvgUXQh+8sC53P7gXEYedzrOp3/L2kCEw+88i9veWElr1VqGTTqEqbkxlv9nAZ9XtpLu1Bh6whDk2OOZvbya2vWbiQaa8aTnkl2Sx2HDcyjNcCHLVxJavYzGNbXUNoc6hFm6AJ9DI8ul28IsL57sdFLyMtHSssGXjfT4CcZMgjHTiuVHbGFWKEZrKGoJs6JWMw1LmGUaXYunmObWBVR6IllhlmLbCETXXNl2WlLPJ8SJQohVQogyIcTNPVy/TgixQgjxlRDiPSHEoIRrhhBiqd1mdb93V1AzfYVCoUikF102hRA68BBwHLAZWCiEmCWlXJHwsC+AiVLKdiHElcAfgfPta0Ep5bhe6YxNv5jpKxQKxZ7Ciukn15LgEKBMSrlOShkBZgJnJD5ASvm+lLLdPpwPFPfin7MVatBXKBSKBOI2DMk0IEcIsSihXd7t6YqA8oTjzfa5bXEp8GbCscd+3vlCiDN74+/rF4N+W1YRw448nVcmTmNla5hPJl9Fe30l/3f6IJ7/bDNn/2wy1762grbqDRx/yljc857gwzUNlKY4OXRsPo5jfsST8zfSuO5LNIeLvKEjOHH/fHLaK6mbv4SasgYaowZBw1qjX+Cx1uhnjijBOXAEQXcmW9oibGpsp6opSLAtQigQsdfoB3tcoy903Vqj7+xco687HHY8v+eC6LoQHfF8l0PDpWs49c41+s6OuH6n0VriGv0uhmti9wqia9uI+Se7Rr+v+fw/N7A2EOV7T23h+CsuYcbZ93DF7Sew5sQbqVnxCf931WG8fsdsJmV5Mc7+FR+9uRhvZgE/O2Ukodce4bOyRipDMSZkeBh0+tGsbNX45MsqWqvWApCaW8KAgRlMKEwnLVRHuOwrGr7ZSOP6JraEDNpi3Quia6TkePFm+0jJy8SZkYGemYvp8WO4fQSiJqGYacXz7TX6bWFrnX4wFCMWTVyfb2KasuNz1X2NPmxdEH1n1+irmP92EFj/v5JoQJ2UcmJCe2yXX1aIi4CJwP0JpwdJKScCFwJ/EUIM3Z0/Dfpw0BdCeIQQnwshvhRCLBdC/M4+P1gIscBOajwvhHD1VR8UCoViZ4lPlnopkVsBlCQcF9vnur6mEMcCtwGnSynD8fNSygp7uw74ABi/y3+YTV/O9MPA0VLKscA44EQhxCTgPuBBKeUwoBHr54xCoVDsJdi/ppNoSbAQGG5Pdl3ANKDLKhwhxHjgUawBvybhfKYQwm3v5wBTgMQE8C7RZ4O+tGizD512k8DRwEv2+SeBXolTKRQKRW/QmzN9KWUM+DnwNrASeEFKuVwIcacQ4nT7YfcDPuDFbkszRwGLhBBfAu8D93Zb9bNL9OmSTXu50mJgGNaypbVAk/1GwHaSGnZC5HKA7IIiUvqyowqFQmEjbC1MbyGlnAPM6Xbu9oT9Y7dx36fAmF7riE2fJnKllIa9xrQYa+nSyJ2497F4cqSxNcqSu47iw7p2fnnjUVz2u9c4dNr3WXn5dLJcOoW/+Rtvv/whWUPGcsfxw1nyxxfZEopx2AG5jLl0Kp/Va3y1uJJg4xZ8BaXsNzqXw0rSiX39IZUL1lHWFu1IzGU6dQqzvWQOz8VTOhQjfQD1wRjlzUE21rfT0hSivTVMONBGJNCMEQltlcRNNFiLbzWnC03XcDh1q7msbaIgq0sS19GZtNW0uBCLDsFWZyWtrslb2E5FLCGSFmbtLskLV3bt+TdOPZpb593H4hef4bVjdFa3hWm4+A9ccO/7DDrsNPZb+G/mNwQ56aZjuWPuWupWL2TwpMOZNjKDr//1PuXBKD6HxsijBuGYfCavLNtC5ZoKQs21uP1ZZJWUMGV4DsOyPIjNK2hYtp7GVVXU1gVpjBpETJkgzNLwZXpIzU/Fm5uJKzsLPTMPzZ9lCbOiljCr2U7etoUtYVYwEiOcIMyKRU2rYpaUHdWyzFikizALVBJ2TxBfFLGj1h/ZI+IsKWWTEOJ9YDKQIYRw2LP9HpMaCoVC8W2ifWvr0vqevly9kyuEyLD3vViKtJVYsalz7YdNB17rqz4oFArFziJQM/1dpRB40o7ra1gJjNlCiBXATCHE3Vjy43/1YR8UCoVip9mHC2f16eqdr6SU46WUB0opD5BS3mmfXyelPERKOUxKeV7imtRt4fD6mLf/4fzyl4dTfeUD1K1eyFs/PYSnX1nFDy4ex/Vvb6RpwzKOPG0yeYue573FVQzwOBh7+dF4T72MRz9ZT+2qxWgOF7nDRnPmuCIKo7XUfTKf6mW1VIetvLJXFxR5HWQOySBrZCmu0pGEU3MtYVZTkI11AdpbrHh+NNCMEQkSC/dgtpYgzNIcLnSXF4fLjcOlbyXM8rr0HounxIVZTs1utjDLqXUKszqKqCQIszoKqWBdi5utJVM8pb8IswDeWl3PSQvzmHzRj3jm0Olcc+3hnH3PPDZ9NptHf3k4b1zxBGPTPaRdcz///e9i3P4srjh9NMbsf/Dp0mq8umBsupvh501ldSyDdxZX0FKxGgBffikFpRlMHpRJdqyRyOovqF9ZQf2aRraEYl2EWWkOnSyXTmpeKql5flLyMtEz89Az8zC96ZhuP+1Rk2DUpDkcozVi0Nwe7YjrR8JdhVnxrTS3XzylJ2FWTzF/JczaBZKc5e/zM30hxGFAaeI9Usqn+qBPCoVC8a0hSHoNfr8kqUFfCPE0MBRYCsSnCRJQg75Codjn2JfDO8nO9CcCo6WUsi87o1AoFHsD+/CYn/SgvwwoAKr6sC8KhULxrbOvl0tMNpGbA6wQQrwthJgVb33ZsUQOKEnnzfIWqq/+K2fd8l8mXfgD1lxyDj6HxsA/PsFLz75P1pCx3H/6aJb8/kkqQzGOOjCPlNMvZ35rKgsXbKa9vhJfQSmjx+RzxKAMzK8/oOLTMla1RmiLmXh1QY7LQWG2l+z98vAOHY6RWUJNe4wNjUHW1QY6hFnRQHNSwiyHy4vu8iYtzEpsyQiz4ola6CrM2tZP031FmAVwz7x7+Ojf/+b9s3wsaQoRvOEh1n88i4GTT2Xyiud4tybAmTcdw81vrqF62YcMOexofnxgLl/8fQ5rAxEmZHg48OhSnEdN4+VlVVSstsR7bn8W2YNKmToqj1E5KWgVK6hbupr6NY3U1ASoi2xfmOXOy7GEWek5mN502mOSQIIwqyUUpTUUoy0UtYVZVvI2LswyDNMSZEUj2xBmmUm/Ryphu+uoRC78ti87oVAoFHsT/cJzfhdJatCXUv5PCJEPHGyf+jzRDU6hUCj2FUQvlkvcG0nqC00I8X3gc+A84PvAAiHEudu/S6FQKPonKrxjmfsfHJ/dCyFygXfptEjuU5q/XslNt/+IiTc8S+OGZax9+CxuvmklV189mSteX0fDui+58Mafk/vpk8z4vJLSFCcTrjmRj5q9PPRhGTUrF6I5XOQP35/zDiqmKFJF1QcfU/l1TYcwK8floMjrIHtYJtn7D8E1ZH8CqblUbGlnfUN7F2FWJAlhlu72dhit9ZUwKy7GShRmJVbMShRmJU5c+rswC+DoT/OZ+pNLeeKgi7jhN8dz+O3vMOSIM3j6+iN4acJhHJzpIeWaP/HSZU/jSc/lF+ceQPSl+/lgyRZ8Do1xxw1m2PnHsTKazpwFq2nasAwAf+FQioZkcnhpFjnRekLL5lO7bDM1NQEqgtsXZqUWZqNn5iEy8jA9fkuYFTR2IMwythJmbatiVqL4KhlhVk+oOP+OEajwDoDWLZxTz779vigUiu8wfbXIYW8g2UH/LSHE28Bz9vH5dPOHVigUin2C7ayA2xdINpF7oxDiHKxyXQCPSSlf6btuKRQKxbeDAHqxhspeR9IhGinly1LK6+y2Rwf8NsPkozNvp3nzas646hIWn3YmAzxOMu58gllPzSZv9BQeOG0k83/zFFtCMaYeVozz9Gv407trWDK/nGDjFtKKRzBhQiFHDsoguvgdyj9a07FG3+fQGJjioCgvhezRA/AOG0ksayDVgRgbmqw1+s0NQQItISKtDcRCAaKhwFbx/Pgafd3l7dg6XO7O9fkJa/S9Lh23HddPcel2fN+K51vxew2HrnWs0XfqPZRrsyPrWkJsv6M/PRit7cwa/V39ebsn1ugDLHzhWV4fU055MMrKC++mYuEcXr11KsPfup9P6oOce/+5XPnyMmq/mc9+U4/hoqEuFv5pDuXBKJOyvAyffib61B/yfwvLKV++jlBzLZ70XHIHD+KEMQWMzk2BDUup/WINdavqqQjGuhRPSXfqZLk0/Nkp+Af4SCnIxp2Xi55dYBmtpWQSiEnaoiYNwSjNoRiN7RGa2qM0tUcIhiyjtZi9Vj9eSGX7xVPM7Rqo7choTZE8QoikWn9ku4O+EOJje9sqhGhJaK1CiJY900WFQqHYc1gLIZJrST2fECcKIVYJIcqEEDf3cN0thHjevr5ACFGacO0W+/wqIcQJvfH3bTe8I6U83N76e+PFFAqFoj/QW3N4u57IQ1hFpDYDC4UQs7oVOL8UaJRSDhNCTAPuA84XQowGpgH7AwOAd4UQI6SUu/UzLtl1+k8nc06hUCj6Pz2EUrfRkuAQoMyuIxIBZgJndHvMGcCT9v5LwDHCih2dAcyUUoallOuBMvv5dotkY/r7Jx4IIRzAQbv74gqFQrHXsXNFVHKEEIsS2uXdnq0IKE843myf6/Exdu3wZiA7yXt3mu2Gd4QQtwC3At6EGL4AIsBju/viyVI0rIArfvkQP7/1Cu4dHeBnP97EvY9eyJlPLCRQW86115+P9tzdvLGslgPS3Iy9/gJeWRdg2fy11JctweHxUXLAaC6cWEJe0xo2zP2IDSvqqAzF0AXkux0UDfCTNTyTnAOH4hh8AM3ODDY1BCirbWNDTRttTSHCrS1EAs3EIsEOAU0czeFCd7rQXR40p53MdXsTkrhax9ZlJ229LgcuvavRmlOzTNZ0gZ3A7TRf6y7Mik80Ek3XenIITDRaS1aY1f3+RLY1v9mTzoS/+9OvuOu4E7n1hV8w6FdPc9B5P8D/yI08fv/7nFacRu3pN/H29L/hLxzK3dPG0fj4Xby/up5ct8648/aHI37Ax5XtzFuwiabylQhNJ71kFCNH5nBkaTaZbeW0fTGfmi83U1UfpC7SKczy6hppDo18jxP/AB+pBRn4inLRswsR6XkYKZkYzhTa2mO0hg2aQzGaw9EOYVZbKNaZuDUksYiBETMxYrEOo7XtCbOALsKsZFHJ3eQQUiKSf6/qpJQT+7I/vc12Z/pSyj/Y8fz7pZRpdvNLKbOllLfsoT4qFArFHkVIM6mWBBVAScJxsX2ux8fYUZR0LAFsMvfuNDtavTPS3n1RCDGhe9vdF1coFIq9DwnSTK7tmIXAcCHEYCGECysx292WfhYw3d4/F5hnF6yaBUyzV/cMBoZjeaDtFjsSZ10HXA78uYdrEjh6dzugUCgUex29VCRQShkTQvwceBvQgRlSyuVCiDuBRVLKWcC/gKeFEGVAA9YXA/bjXgBWADHgqt1duQM7XrJ5ub2dursvtDusi6TgTs/hrtTFvDj5Xk4u8LHmxBv5/Pw7GHbk6dwyzstrF71GxJQce94oWg+7iL/+4zNqV87HiATJGz2F4ycN5MhB6bS/+DAb31/H6rYIEVOS5dIZnOokb0wumSOK8ew3jljOEKraYqypb+ebqhZaGi1hVrjNEmbF465xNIerQ5zVUTzF7cXhcnYKslydAi2XQyPF1UMRFV3riOE77P3tCbO0jjh9YjGVHRutdRFs9fB+97XopDeeftpbd/Op381t8miC9f/HB9dcyt05l9MWM7n2pT/yvUcX0Fq1lhOu/AnHOTbw2gPzqA0bnDMym9JLL+H1dS3MXFROxfLlRAPN+PJLGTC8iJPHFDIqx4Px2QKqF31D3Tf1HUZrhowbrWnkunVS81PwF/rwFeXiyi9Ezy3CTM0i6vDSHjFoi1jCrJZwjOb2KE1BS5gVDseIhg1LmBUxrMIp8eIpcXFWouGaaXQRZpk9iLB2JMxS8fydQMpkZ/FJPp2cQzfbGinl7Qn7ISwH457u/T3w+17rDMkv2TxPCOG3938thPivEGJ8b3ZEoVAo9hZ6Maa/15Hsks3fSClbhRCHA8di/Rx5pO+6pVAoFN8WEsxYcq0fkuygH/9teAqW2dobgKtvuqRQKBTfIpLeTOTudSRrrVwhhHgUS0p8nxDCzR7002+uqWXpQ5fytxEHs6E9wt+/eYYR976P0+vjn1dNZu0tl/FuTYBTC/2MuOVWbv9kI2ULlmBEgqRkD2DYxGFcOKEI14r3WD57ASs2NlMbjuHSBCVeJ4XDs8gdO4S0EUMQJaOoizlZXd/CisoWKmsDtDYECTfXEg00dxROicdIhaYjNB2H24vu8qC7rWLousubYLBmr9F3abg7DNYceJ0JRmvxNfpCdBRQsfY19I5zWo9r9OPF0LcVKo/H+K39TpO2RPbUGv3eShfc+8f/8bemRVxy7K/5zb3X8flxJ6MLwSXnjWKmcyJfvfFHBhx0Ag+dO4YVV5/P+7XtjPK7GX/lUWwZdDgPPbuUDStqaK1ci8PjI3voGKaMLWTKwAw8lV9RvWAB1V9uYX1LmLpIDMPO6/kcGrluB7npHtKK0/AV5ZBalIueW4T052CmZNIaMQlETWttfjhGfXuE+rYIze0R2kJ2PD/aabRmGNYa/fiafMOO7SdqQRILpwA7vUZfsTNI2IkC9P2NZAfu72Nln0+QUjYBWcCNfdYrhUKh+BbZl2P6yfrptwsh1gIn2E5vH0kp3+nbrikUCsW3RD8d0JMh2dU7vwCeAfLs9h8hxNV92TGFQqH4VpASTCO51g9JNqZ/KXColDIAIIS4D/gM+HtfdUyhUCi+Lfpr6CYZko3pCzpX8GDv7zF3rdSsbLQbLyRgmFxz2QSu+drPps9mc+xFZzBp/eu88MwyBngcfO+uM/hEDOXFOato3rSStOIRFI45hEuPHMpIrYHq2bPY+FE5G9qjGNIyWhuSl0LBQUWkjxuHe/QhhDIGsrE5xKraNkuYVR+kvbmFSHuzJcyKdTVaE5qO5nShOZxozgRhllPH6XbgdHc1XPPaSVyX3inM8rp0nJolxooncZ26ldi19hMM17ROYZawK2bF/4F6Emb1lDjdntFaojBrb03iAvzq2sMY8+uPKT74eK4LzuWZ+RVcfttxDPv3f7n1wXfRnS5uuuxQcub+nTdfW4Mu4MjjSkmfdjUzFlewetF6ar9ZhBmLkF48gkGjcjl1/3wGimZCi95jy4I1bF7XRHU4RtCwqmV5dUGmU6fAo5NW7Cd9UCb+gfk48geiZw/ATM0mYOq0RAzaIgZ17VEag1Ea2iI0B6M0tUcJB6PEIkZHMtdK4nYKszrM1oyuwqxE4klcJczqK3rVhmGvI9mZ/r+BBUKIeJnEM7HW6isUCsW+Rz8d0JMh2UTuA0KID4DD7VM/llJ+0We9UigUim+LXrZh2NvYkZ++B/gpMAz4GvinbfKvUCgU+ySCfTumv6OZ/pNAFPgIOAkYBVzb153qzoh0yT+eXc6fn72Mzcdcw5Pn3snAyafy7Hn78e7on1AdjvHT80ejXXAbv354AZu/+B/O1HRKJ4xnyrgBnDYii+gbf2XN61+xtClEW8wk3akxzOekYFw++YeMxjniIGKZxWxujbKipo3lFc001AZoawoSaq4lGmjpEGbFiZusOWwxltPjw+H14fR4cLkdCYVT9I54viXM6tx6XbpttCYSYvjbN1oTCfH8uDCrezw/kZ6M1npie/H8bbEnC6ck8urZd7Hphgeoeu9+Hsgby1nDs2i67D4ufHgB1cs+ZMr0i/lJcYC3z3uOtYEIZwxKZ/QNlzOvMYWX31tB3aqFxEJtpGQPoGj0cKYdUsLBA3yw+D0qP/qCLUurWR+I0hCx4uFeXcPn0Cjw6GQU+kgr9uMfmI+npARHwUAMXw4Rl5/WoEFLyKA5HKMxGKWuLUx9IEJTe4SgLcyKhGMdZmuxSNQSXdkmftsyWuswYdvJeL5iV5CwD4vfdjToj5ZSjgEQQvyLnfByFkKUAE8B+VjC5seklH8VQmQBzwOlwAbg+1LKxp3vukKhUPQBcRuGfZQdrd6Jxnd2IawTA66XUo4GJgFX2dXdbwbek1IOB96zjxUKhWKv4busyB3brTZuvFauAKSUMm1bN0opq4Aqe79VCLESq6jvGcBR9sOeBD4AbtrVP0ChUCh6l+9wIldKqffGiwghSoHxwAIg3/5CANiCFf7p6Z7Lsap2kS4c/PPYw3lq8EXcd+ub6C4Pz908lTU//QGvb27hzCGZjP7DPdw4dy3L3vuEaKCZkkNP4YfHD+e4oTmkLn+H5S98wLI1DVTbRmv/396dx8dVlg0f/12zTxaSJgrXMjkAACAASURBVGnTvWm60N0CBSlLoaUs1SKIPoKPiPqAiK/66kdBtvf1URFFEUEfQagiiCIghbIIUrZCKbKV0pZC6b4lTZql2TN77uePc2Y6STPNlLaZmeb6fj7nkznLzDkH0jtnrvu+rrsiz8OoyWUMO2kivmknEyk/loZAjA/qWlm9q4Vt1W207Q0QaKoj0tFCJNDeezw/RaE1t8+Jxx6n73Q58Ptc+xVaixdbczsEnz1uPzE+v0ehNbdzXyzf6egez+8tqr5vHH/iv2diO+w/Rr/PeP8B9/btcIf+r//eLdz2+xtYdcqZxIxh3vJHmXbzy+x8+wVGz17IQ187gXX/dRHPVrcy7Rgvs2/4NDUTz+WWv65i56q3iAbbcfkKKJ88i3mzRnJWZQn5VauoffVVqt7cxZamYKLQmschlHmcFLmdDC7yUTymiKKxQykcMxxX+ShMUTld+aW0hbtoDcdo6AzvV2ituT1MOBglEkouuBZLFFbriob3K7TWM55/IDo+/zAbqI3+4SAiBcBjwPeMMa3JjYsxxohIr/OSGWMWAYsARjh8h2fuMqWU6ku8DMNR6oiWRxYRN1aD/6Ax5nF78x4RGWbvHwbUHclrUEqpg2Mw0Uhay6EQkRIReUFENtk/B/VyzEwReUNEPhCRtSJycdK++0Vkm4istpeZ6Zz3iDX6Yj3S3wusN8b8JmlX8szvXwGePFLXoJRSB83QXwXX0hnU0glcZoyZCpwH3CEixUn7rzHGzLSX1emc9EiGd04Fvgy8LyLxi7kBuAX4h4hcDuzAqtWvlFJZwWD6a5KaPge1GGM2Jr3eLSJ1wGCg+eOe9Ig1+saYFaTu/zvrYD7L5YChDz/Ndf/xc4It9dx4y9VMeO5WfvaP9Uw7xsuZd36TR1sGs3jJy7TVbKF0/PGcM388l84YSnHjRrb//WE+XL6Lje1WR+wov5tjRx/DyFPGUfzJ2XSNmcmOtgjVrSHW7m5lfXULzfUddOxtIthST7izlVg40G22rJ6duPHELKvz1pU0a5YTj91pW+Bz43fvS8zyuBx2B64Tl3NfJ65D9i+0JgJOu4haz07cvgqt9dWJ21M2F1qLO/5zl/DZ52/hpvfruH3Jd1mwuIZtK56icNg47vzuacg91/HYM5sp8Tg578ufwHfpjVz/7GY+en0tHfW7yCsdTuGw8cw8YTgXzxzBqPBu2l77F7te/Yjt25rZFYgkCq2VeJwM9bko87oYVFlM0dgyisaNwDV8LI7Bo4kWltMac9ISilLfEaahM0JLKEJ9a4i9HVZnbjgUtZKy7Nmyenbi9lZoDXokX8VimozVHwwHM3NWmYisTFpfZPdHpiOtQS1xInIS1jS1W5I23ywiP8L+pmCMCfV10iPekauUUrnloDpyG4wxs1LtFJEXgaG97Lqx2xkPMKjF/pxhwF+BrxiTGFp0PdYfCw/WoJdrgZ/2dcHa6CulVDJjDrmTdt9Hmfmp9onIHhEZZoypOdCgFhE5BngGuNEY82bSZ8e/JYRE5D7g6nSuqd8mN1dKqdxgetQ/Sr0coj4HtYiIB1gCPGCMWdxjX3wUpGCVu1+Xzklz4km/bOoETvveYly+fE67cAHXF2/gju8vxu8ULv7xp9g442Ju+vVy9ry/nILyCmbMPY7vz6mkcPVT1K14jY8e/5A1LUHCXYbhPhfTyvyMOnU0Q04/CZn4SXbH8lhT28qOpk5W7WiisbaNtr2tBJpriXS2EgvtH893uD37xfPdXg9urwuPd98EKn6fC4/LQWGPeL7f48TncnafOMVOynLYxdbiSVk9J05JN56fXHzt406ckkom4/kAr85t5v+espQffu8Uflu0kBU330blnAv48mcmc8aWx7jnZ8/TEuni0rPHUnHDTfzu3Vr+9dxH7N26Bnd+EUOnnsiwsYP46sljmF4YJvzS02xf+i671taxpSNMS8T6Bl3kdjLc52JYqZ+CIfmUjC+laNwIvKPG4hpeSbRoKAGHj+bOGPUdEeo6wtR3hGjpjFDXFqKxPUQwECEcsBKzrLh+jGg4RMwu4JdIzEokZe1LzAK6FVqL04lTjqD46J0jr9dBLSIyC7jKGHOFvW0OUCoiX7Xf91V7pM6DIjIY65/1aqyKyH3KiUZfKaX6jzmYjtyPfxZjGullUIsxZiVwhf36b8DfUrx/3sc5rzb6SimVzNBfQzYzQht9pZTq5uguw5ATjf6He4I4tq3hvruu4aKyNh6acSW7gxG+ddWJhL/6M77+P/9m6+vP4S0sYfKZp/GzhVOobHiXDX98kN3v1vJmfQctkS5KPE6mF3kZM2c0I+adhGvGHBp8Q3h/dzsrdzSxo7GDmupWWho66WysJtzW1K3QWvL4fIfL0+vEKV6/C4/fjdfvwut1UWjH9HubOMXnsoqseeNj9JPG6fecOKXnWP1U8fy4A8Xzk/UVz++9mFtm4/kA//+Ma/jK3DFsuuoObv76ryibeCJP3DCXCY2rWHzm/7C+LcQXpg/hhN/8iEfrC/jj4++y5/3lOFwehkw5lbmnV3D6uFLmjjmGrtf+zs5/rWDXiio+bA0lJk4pcjsY7nMxqshL6fhB5JfnUzxxFPmVlbhHTyR2zFBC3iL2dkZpDESoaQ+xpz1EbXOQtlCUxvYQbR1hQoEooWCESHDfxCnJ4/OT4/nWeP1DmzhF4/mH6DCO3slGOdHoK6VU/9EnfaWUGjj6b/RORmijr5RSSQwG0w+jdzJFG32llEqmT/qZF2pr5te/+DbzXrmNpbe+wJt7A1x18RTKf/UXFv7hLdY9/ywOt4eJZ8zjJ5+bzgnRLWy96y7efXYz2zoi1IdiFLkdfKLIS+Wc0Yw+90S8J55Dc9FY1tV28Ob2vbyzpZHO1hBNe9rpqN+5XycukOjEdfnycbg8ePKLcOcX4cnLx+tz4/G7EslZXq+LAp+LAp/bSs5KrFszZ3ntGbPiCVm++LozXnDNkSi4lkjIInUnblzybFnQeydub7NlHe4ia0favIpiiv7+NJ/+6h34BpXz4E8voPDua3juT2+wrL6TC8YUcdo91/Kicwq/eGAlO958EdMVY8jUU5l92hi+MXsM4wZ5cax8kp1PLWXbi9tY0xxkT8iaLavA5WC4z01FvoeSCSWUHFtO/rBSCiaMx10xmVjxCEL5g9kbiNHYGaWmLURdh9WJW9MSoDMco6U9TLAjQihgdeKGQ1EioTCxUIBYOJDoxE102monbnYwBhMJ931cjsqJRl8ppfpP/yRnZYo2+kop1dNR/I1JG32llEpmzFEdJsuJRn/oiHIu33gfP796CS2RLr5xwUTG3/c4Cxe9w8olT2JiMY6dt4AfXzKTuZ7dbPvNr3n7kXWsag4SiBk7nu/j2NNHUbnwk/hPWUhL6UTW7Ongta2NvLGpgfqqVoKdYTrqqwi1NBDuaCEWDiSuwenxJ+L5Ln8BTpcHl6+gWzzfaydl+fxuCnwuivM8FHhdeF0OK5bvcSbi+dbkKdYEKvti+1Ys3yHSLZ7vdLAvQYve4/kO6R7PT07WykQ8/0iH/if8+1VO+tqdiMPJA7+8jEmP/YTf/+JF6kMxFg4rZN79P+SNIWdw3Z/fYfOKF4iFAwyZciqzzzyWH8ydwDRHPV3vvceuxU+w+V+bWFPf2SOe72JcgYfBU8sYPG0YZTPG4y4tw1Mxia7SMUQKh9LYGaWhM0pVa5Da9hDVewPUtASpaw0RDscIdlrx/HAgmjKer0lZ2UlH7yil1EBhDCamjb5SSg0Ixhi6ItFMX8YRo42+UkolM+iTfqYNCTZw01V/Z1y+h1POGcu4+x9nwR/e4p3HnsDEYkye/yl+funxzPdUse3WW3jj4fd5pylIzFjx/OOLfUw6cwyVCz9J3pwLaS6dyHu1HbyyuYHXN9RTX9VKc80ewp0tacXzPXlFOL3+tOL58YJryePzk+P5yePz42PzHXL44/npTpqSC/F8gFmX3o7D7eHR313JpId/xG9veh6nwPkjj+Hsh/8fy4fM5ep732bjsueIhQOUT5/DafMm8cOzJjBN9tD+9F9oWLuZTf/cwJr6TnYFIt3i+RMLvd3i+b6J03AOGkJXWQWRwqHUd0ap64hQ1Rqkui1I9d4AVU2d1LWG6GgLE43E0ornJyZE13h+VtFGXymlBghjDF1aT18ppQaOo3n0jk6MrpRSyezRO+ksh0JESkTkBRHZZP8clOK4mIistpenkraPFZG3RGSziDxiT6LeJ230lVIqSXz0TjrLIboOeMkYMwF4yV7vTcAYM9NePpO0/ZfA7caY8UATcHk6J82J8M7uXU2cOLiEzy39DXsqTuesX69g7TNP4PYXMH3hudzxn8dxfPsaNvz3bax4ehNrWoIATCzwMjrPxbHnVFJx/ul4Zn+a+sIKVla1sXxzA29trKeuqpXW2lo6G6uJhYOEO1p6nSnL5cu3i6tZRdacLgdenxtfvrvbTFnFeW4KfO5EJ25BfOYst5M8uyM3PlNWb524TsfBz5TVW8cu9H8nbn/WYvMPGspLd1yC66YruPXudyj3urjs+vmUX3Qxj0Ym8JO73mD7v5cCMPyEczl7/ni+f0YlEwJb2bvkL2x8bCVNW5tZ1RRIJGUVuR2M8rsZN8jH4ClllE0fSdmMcXgqp+IYNYkubyHB/MHUd0So64iwsyVIjd2JW9MSoKY5SLAjQrDT6shNVWQtGg5gYjHtxM1iXf3TkXsBcKb9+i/AK8C16bxRrH/I84D/THr/j4E/9PVefdJXSqlk9pDNNMM7ZSKyMmm58iDOVG6MqbFf1wLlKY7z2Z/9pohcaG8rBZqNMfGvG1XAiHROmhNP+kop1W8OLiO3wRgzK9VOEXkRGNrLrhu7n9IYETEpPmaMMaZaRCqBl0XkfaAl3QvsSRt9pZRKYjh8o3eMMfNT7RORPSIyzBhTIyLDgLoUn1Ft/9wqIq8AxwGPAcUi4rKf9kcC1elcU040+oPy3Fy07lm+9nwDb//5BbateIrCYeM45cJ5/O6iaQxfu4R3f3U/K16vYmN7GL9TmHaMl+knDaf02CGMWDAP5/HnsMtRylvbm3l5Qz0fbN1LQ3UrrbVVBJpqCbU1JQpfgRXPT07K8uQX4c4rwpNfiMfvxul04Mt34/W78fhc+H29x/P9HiduhxW/j8fzfS4rpm/F9vfF87vF8NOI58dj6OnE86VHwD2X4/kAm++7jPfOPYcHlu/k5BI/X7jrMraf8S3u+6CWe/76MnveX44nv4jRs87g4gUTuXzWSMp3vs7uRx9hw5K1rN3ZQlMkRn0ohlNgsNfJKL+bsUPzGTyljMEzKhg0ZRye8TNg6DiixSPpjBoa26PsbgtR3RqkujVI1d4AtS0BGlpDBDsjBDvChAJRYrEuIqEokWAwEc+PRa1krH1F1iLd4vYHiuenittrPP8IMIaucL+UYXgK+Apwi/3zyZ4H2CN6Oo0xIREpA04FfmV/M1gGfB54ONX7e6MxfaWUSmagq6srreUQ3QKcLSKbgPn2OiIyS0T+ZB8zGVgpImuAZcAtxpgP7X3XAt8Xkc1YMf570zlpTjzpK6VUfzH0T5VNY0wjcFYv21cCV9iv/w1MT/H+rcBJB3tebfSVUiqZIRFmOxrlRKPvnTCR2XduYN2zS+iKhhl23Hyu/NIsrj55GJ0P3Mzy373A8m3N1IdiDPY6OXGQnwnnVTJm4Rw8FZOJTZrD+pYulu9oYNn6OrZva2Lvnnba92xLFFjrOQG60+vH5fHjzj8Gt68gMQG6L8+D1+/CYY/T9/pdFOa5KfS5KPJ7KLTj+AU+F/keFz6XNSmK12XH9XvE8ZMnQI+PzXdgx+/tuP6BxuZDj8JrSf/dDiWen62x/LjFI4/j9cYAl5wwjDkP386DrSP52c0v07DlA9pqtlA4bByT5szmOwuO5cIJxbD8QTY98k82PbeV1UkToHscQrnXxdh8NyMri63x+TPGUTh5Mp7KqURLxhDKK6W+I0ogYhIF1qqaAlQ1BahrDdLcFkqMz48EY4SCEUyXIRLsJBbaNzb/QBOmgNXQ6Nj8bGC0DMPHISJ/FpE6EVmXtC2ttGOllMqYgxunn3OOZEfu/cB5Pbalm3aslFIZYYwhFo6mteSiI9boG2OWA3t7bL4AK10Y++eFKKVUVjF2+K3vJRf1d0w/3bRj7HTmKwFGjBzVa0qbUkoddjpz1pHRR9oxxphFwCIA96DRpu6pRxj6ibmMmjQiUWBt47ev5rUnNiYKrE0u9HLc5FLGn/8JBp+7gK7JZ9AQc7FyZ3uvBdZCbU3EwoFEp9iBCqxZM2PZs2T53Lg8jpQF1gp8Lnt2LKvAmlVULXWBtXgSVqLTto+ErN46cOHoLrDWU3Ugyo9uWoD3u7dx0SNrWfHk32it2ojT42fEiZ/qXmDtnl+z8bGVrF1Xz5aOMO3RLjwOocAlKQusOUdOJDJoNE0xF40t1gxZLaFoygJroUDUSsYKRYkEOzGxWMoCa/GkrOQOXEivwJp24PYDAyaWsmnKef3d6KeVdqyUUpliMP1VZTMj+jsjN552DAeRNqyUUv3GgOkyaS256Ig96YvIQ1i1ostEpAr4b6w043+IyOXADuALR+r8Sin1cRgDsfDRG0Y7Yo2+MeaLKXbtl3bc52fFopxx+X9x5xdmMNbdSfO9P+a53y5j+Z52WiJdDPW5OLEsj/ELxjP6M2fhmnUeNZ5y3tnWyo7mAMvW11G1o5m9NU101O8k1NJAJNC+32QpDrcHty8fl78gEcv3+P12PN+VmCwlz+/G43JQnOfZr7haPCErubiaQ6w4vhXTt2L5DumekHU4i6vBgWP5ye9Jlgux/LirNzzB3bvyuO0Hz7L73aW4/AVUzrmAoRXFXH3eJM4Z7iS27F4+fOQFNry8g3WtIWqD1hC7Eo9VXG2w10l5ZTFDppdTNmMc+RMn4Rk3neigkbR5imkIxKwYfluQ3a1BWjoj1LQEqWkO0GonZIWCkX3xfLu4WpddWC3Wrbja/glZOllKljJGY/pKKTWQdGmjr5RSA4QO2VRKqYHDAF052kmbDm30lVIqmTHakZtpEyqGsHRelI3XXsryt2t4dcte6kMxSjxOzi3PZ8JZFVReeAae2Z+mobCClbvbWb55B29trKejNURjTRsd9TsJNu3pVlGzZzKWw+WxZsiyK2p6fW58+W48fjcerxOf351IxvK4HBR69yVj+d1O8tzORAducjJWvCM3VUVNpyN1By7QbRvs34HbbdtR3oEbN/22LWz/91IAhp9wbiIZq+IYN7z2d7be9jSbn93CqqZAoqJmkdvRLRkrvzyf0qljOWbKJNyV0+gqHUNH/mDqO6PUNdozY7XuS8ZqC0b3q6gZDkWJhMKJ2bGSk7G0Azc3GU3OUkqpAUQbfaWUGkg0I1cppQaOfsrITWd+ERGZKyKrk5agiFxo77tfRLYl7ZuZznlz4klfdm7l9yd/g/VtIQCG+lycP/KY/ZOxqltZ9tYWVm9upGF3K621uwl3tqRMxnL7C3AlJWM5vf6UyViFPhdFSclYHpcjZTKW22nF8r12MpbTQUaTsXK5sFoqO99ZxpiTz+Gz50zgqpNHM7L+PWrvvoZNH+1KmYxVOSSPIVPKKJs2itLp43EMGrJ/MlZtZyIZq2pvgNqWAHuagwQ7I0TDsZTJWFE7nh9PxrIWjeXnIkO/jdOPzy9yi4hcZ69f2+1ajFkGzATrjwSwGXg+6ZBrjDGLD+akOdHoK6VUvzGGrv4ZvXMBVqkasOYXeYUejX4Pnwf+ZYzpPJSTanhHKaWSGGM96aezHKK05xexXQI81GPbzSKyVkRuFxFvOifVJ32llOrhIGbFKhORlUnri+y5QAAQkReh1zmgbux2vj7mF7FL0U8HliZtvh7rj4UHa+6Ra4Gf9nXBOdHo17eEaPHFuGBMESUTShh3/vEMmn8+obEns64+wCsbGlm2fi27dzbTVNvcrahaPKYK4HB5cHr9KYuquTwOvD57ohS/mwKfa7+iagU+Fz6X0yqg5rRi+W5nfGx+96JqTofgwIrXOx3sey19x/Ghx1h9e1uqOH7Pfcnv6SmdWH42xvGTLf7TdZw1VIi+9ACbvvkSb75ixfHbo10EYga/U6jIczO+wMOwCSUMmV5OydSxFEyagnvsVKIlo+nyFlITgsZAlJ172qhuDbK7OUBVU4C61uB+RdW6ol2EQ1Fi4UBiXH5fRdWAxJh90Dh+TjAH9RTfYIyZlfqjzPxU+0TkYOYX+QKwxBgTSfrs+LeEkIjcB1ydzgVreEcppZLZ4/TTWQ7Rwcwv8kV6hHbsPxSI9fR3IbAunZPmxJO+Ukr1F0O/FVzrdX4REZkFXGWMucJerwBGAa/2eP+DIjIY60v9auCqdE6qjb5SSiUzhlj4yDf6xphGeplfxBizErgiaX07MKKX4+Z9nPNqo6+UUkmMgS6jZRgyauiQAq598EZk5tkE/KWs3dPJ8m2NLHt5JfVVrTTVNtJRt5Nwe9N+SVjicOLyF+D25ePOL8LtK8CdX4QvPy+RfOX1u/H4XDhdDory3BT63BTZCVl+jzPReRsvqOZ2SCIBy0rGkv06bzUJ68gacs2lPPJGFevbwuwNx3CKlYQ13Ofm2EIPZRNLGDJ9KGUzxuOfOBXXmMnEBo2kzVlAQyBK7d4wLSGr87a6aV/nbVtbiGBnhHAgahdT25eEZbpi3TpvrY5bTcI6GsW00VdKqYHBAEdxvTVt9JVSqid90ldKqQGiy0BYZ87KrI7SEfyfhpl8uGgDna2hA8bwnR4/nvwiXL58PPlFVmG1FDH8gjy3nXTlptjvThRR6y2G7+uRhOW0J0bpK4bvTJrcRGP4h8+fn9lEicfJuHw388aXMHhqGYNnVJA3ZNB+MfztgSi1bWGqtwWpbt2dKKTWFoweMIafiN+nUUgtVdxeY/i5ScM7Sik1QBiMhneUUmqg0I5cpZQaYLTRz7AdO/fwt1vv6hYLdXr8uLx+/IPKuxVP8/q9ePzWhOZenxunS/ClKJ7m9zjJdzvxuqzYvVPA63ImJjTvOf4+Hq932sHwA01ofijF0zR237df3HsZvonTcI48luigUbQYLw2BGNXhGDtbAtTUhqj+sIGqpp3UtYboaAsTCkYIdkSsuH0oSiwa7TaheW/j7+P9RTr+fuAwRkfvKKXUgGHQ0TtKKTVgaExfKaUGGA3vKKXUAGHF9DN9FUdOTjT6Ln8BY09baM1u5XbsS7LyuijOc1PQW4E0pwOvPcOVlVDl6LODNt0Cacmds6DJVZlwlfN86t4LEVyxl2BnLaFAlHAgQizWRTQcSXTQRu1OWhOLJTpou6KRRCerdtCq3uiTvlJKDRAG6JcpVDJEG32llEpiMDp6RymlBgpr9I42+hk1dXQxr//y3Exfhsoii2//Q6YvQR2tjvKOXEffhxx+InKeiGwQkc0icl0mrkEppXoTf9JPZzkUIvIfIvKBiHTZk6GnOq7X9lJExorIW/b2R0TEk855+73RFxEncCewAJgCfFFEpvT3dSilVCoxk95yiNYBFwHLUx3QR3v5S+B2Y8x4oAm4PJ2TZuJJ/yRgszFmqzEmDDwMXJCB61BKqf10YZVhSGc5FMaY9caYDX0c1mt7KdZY8HnAYvu4vwAXpnPeTMT0RwC7ktargE/2PEhErgSutFdDeX7/un64tv5SBjRk+iIOo6PtfuDou6eBdD9jDuWDGwgvvYcdZWke7hORlUnri4wxiw7l/D2kai9LgWZjTDRp+4h0PjBrO3Lt/3CLAERkpTEmZcwr1+j9ZL+j7Z70ftJnjDnvcH2WiLwIDO1l143GmCcP13kORiYa/WpgVNL6SHubUkodVYwx8w/xI1K1l41AsYi47Kf9tNvRTMT03wEm2D3PHuAS4KkMXIdSSmW7XttLY4wBlgGft4/7CpDWN4d+b/Ttv0rfBpYC64F/GGM+6ONthzNGlg30frLf0XZPej9ZRkQ+KyJVwGzgGRFZam8fLiLPQp/t5bXA90VkM1aM/960zmuO4swzpZRS3WUkOUsppVRmaKOvlFIDSFY3+rlarkFE/iwidSKyLmlbiYi8ICKb7J+D7O0iIr+z73GtiByfuSvvnYiMEpFlIvKhnTb+XXt7Tt6TiPhE5G0RWWPfz0/s7b2mtYuI117fbO+vyOT1pyIiThF5T0T+aa/n+v1sF5H3RWR1fCx8rv7OZZOsbfRzvFzD/UDPsb7XAS8ZYyYAL9nrYN3fBHu5EsjGSmJR4AfGmCnAycC37P8XuXpPIWCeMeYTwEzgPBE5mdRp7ZcDTfb22+3jstF3sTr74nL9fgDmGmNmJo3Jz9XfuexhjMnKBatHe2nS+vXA9Zm+roO4/gpgXdL6BmCY/XoYsMF+fQ/wxd6Oy9YFa2jY2UfDPQF5wCqsLMcGwGVvT/z+YY2cmG2/dtnHSaavvcd9jMRqBOcB/8SamC1n78e+tu1AWY9tOf87l+kla5/06T39OK004yxVboypsV/XAuX265y6TzsUcBzwFjl8T3YoZDVQB7wAbCF1Wnvifuz9LVhD5LLJHcAP2Tfp04HS9HPhfsAqePm8iLxrl2WBHP6dyxZZW4bhaGaMMSKSc2NlRaQAeAz4njGmVZIm6c21ezLGxICZIlIMLAEmZfiSPjYRWQjUGWPeFZEzM309h9FpxphqERkCvCAiHyXvzLXfuWyRzU/6R1u5hj0iMgzA/llnb8+J+xQRN1aD/6Ax5nF7c07fE4Axphkrs3E2dlq7vSv5mhP3Y+8vwkqDzxanAp8Rke1YVRjnAb8ld+8HAGNMtf2zDusP80kcBb9zmZbNjf7RVq7hKaxUaeieMv0UcJk9+uBkoCXp62tWEOuR/l5gvTHmN0m7cvKeRGSw/YSPiPix+ifWkzqtPfk+Pw+8bOzAcTYwxlxvjBlpjKnA+nfysjHmS+To/QCISL6IOYcM9AAAAmhJREFUFMZfA+dg1Z/Pyd+5rJLpToUDLcCngI1Y8dYbM309B3HdDwE1QAQrtng5Vsz0JWAT8CJQYh8rWKOUtgDvA7Myff293M9pWPHVtcBqe/lUrt4TMAN4z76fdcCP7O2VwNvAZuBRwGtv99nrm+39lZm+hwPc25nAP3P9fuxrX2MvH8T//efq71w2LVqGQSmlBpBsDu8opZQ6zLTRV0qpAUQbfaWUGkC00VdKqQFEG32llBpAtNFXGSciMbuS4gd25csfiMjH/t0UkRuSXldIUrVTpQY6bfRVNggYq5LiVKxEqQXAfx/C593Q9yFKDUza6KusYqyU+yuBb9vZlU4RuVVE3rHrpH8DQETOFJHlIvKMWHMu3C0iDhG5BfDb3xwetD/WKSJ/tL9JPG9n4So1IGmjr7KOMWYr4ASGYGUztxhjTgROBL4uImPtQ08CvoM138I44CJjzHXs++bwJfu4CcCd9jeJZuBz/Xc3SmUXbfRVtjsHq6bKaqxyzqVYjTjA28aYrcaqmPkQVrmI3mwzxqy2X7+LNdeBUgOSllZWWUdEKoEYVgVFAb5jjFna45gzseoBJUtVUySU9DoGaHhHDVj6pK+yiogMBu4Gfm+swlBLgW/apZ0RkYl21UWAk+wqrA7gYmCFvT0SP14p1Z0+6ats4LfDN26s+Xj/CsRLOP8JKxyzyi7xXA9caO97B/g9MB6rjPASe/siYK2IrAJu7I8bUCpXaJVNlZPs8M7VxpiFmb4WpXKJhneUUmoA0Sd9pZQaQPRJXymlBhBt9JVSagDRRl8ppQYQbfSVUmoA0UZfKaUGkP8FDDDoBdR2Fq0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pos_encoding = positional_encoding(50, 512)\n",
    "print (pos_encoding.shape)\n",
    "\n",
    "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n",
    "plt.xlabel('Depth')\n",
    "plt.xlim((0, 512))\n",
    "plt.ylabel('Position')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Masking\n",
    "\n",
    "maksing分为两种：\n",
    "\n",
    "* padding masking，为了屏蔽对齐的时候补全的位置的词语的影响\n",
    "* look ahead masking，为了屏蔽未来时间步的词语的影响"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "padding mask test:\n",
      "(3, 1, 1, 5)\n",
      "tf.Tensor(\n",
      "[[[[0. 0. 1. 1. 0.]]]\n",
      "\n",
      "\n",
      " [[[0. 0. 0. 1. 1.]]]\n",
      "\n",
      "\n",
      " [[[1. 1. 1. 0. 0.]]]], shape=(3, 1, 1, 5), dtype=float32)\n",
      "look ahead mask test:\n",
      "(3, 3)\n",
      "tf.Tensor(\n",
      "[[0. 1. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 0.]], shape=(3, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "def create_padding_mask(seq):\n",
    "  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
    "  \n",
    "  # add extra dimensions so that we can add the padding\n",
    "  # to the attention logits.\n",
    "  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n",
    "\n",
    "def create_look_ahead_mask(size):\n",
    "  mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
    "  return mask  # (seq_len, seq_len)\n",
    "\n",
    "print('padding mask test:')\n",
    "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n",
    "padding_mask = create_padding_mask(x)\n",
    "print(padding_mask.shape)\n",
    "print(padding_mask)\n",
    "\n",
    "print('look ahead mask test:')\n",
    "x = tf.random.uniform((1, 3))\n",
    "temp = create_look_ahead_mask(x.shape[1])\n",
    "print(temp.shape)\n",
    "print(temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scaled dot-product attention\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scaled_dot_product_attention(q, k, v, mask):\n",
    "  \"\"\"Calculate the attention weights.\n",
    "  q, k, v must have matching leading dimensions.\n",
    "  The mask has different shapes depending on its type(padding or look ahead) \n",
    "  but it must be broadcastable for addition.\n",
    "  \n",
    "  Args:\n",
    "    q: query shape == (..., seq_len_q, depth)\n",
    "    k: key shape == (..., seq_len_k, depth)\n",
    "    v: value shape == (..., seq_len_v, depth)\n",
    "    mask: Float tensor with shape broadcastable \n",
    "          to (..., seq_len_q, seq_len_k). Defaults to None.\n",
    "    \n",
    "  Returns:\n",
    "    output, attention_weights\n",
    "  \"\"\"\n",
    "\n",
    "  matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n",
    "  \n",
    "  # scale matmul_qk\n",
    "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "  scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "\n",
    "  # add the mask to the scaled tensor.\n",
    "  if mask is not None:\n",
    "    scaled_attention_logits += (mask * -1e9)\n",
    "\n",
    "  # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
    "  # add up to 1.\n",
    "  attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n",
    "\n",
    "  output = tf.matmul(attention_weights, v)  # (..., seq_len_v, depth)\n",
    "\n",
    "  return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights are:\n",
      "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n",
      "\n",
      "Attention weights are:\n",
      "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n",
      "\n",
      "Attention weights are:\n",
      "tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)\n",
      "\n",
      "Attention weights are:\n",
      "tf.Tensor(\n",
      "[[0.  0.  0.5 0.5]\n",
      " [0.  1.  0.  0. ]\n",
      " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor(\n",
      "[[550.    5.5]\n",
      " [ 10.    0. ]\n",
      " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "def print_out(q, k, v):\n",
    "  temp_out, temp_attn = scaled_dot_product_attention(\n",
    "      q, k, v, None)\n",
    "  print ('Attention weights are:')\n",
    "  print (temp_attn)\n",
    "  print ('Output is:')\n",
    "  print (temp_out)\n",
    "\n",
    "np.set_printoptions(suppress=True)\n",
    "\n",
    "temp_k = tf.constant([[10,0,0],\n",
    "                      [0,10,0],\n",
    "                      [0,0,10],\n",
    "                      [0,0,10]], dtype=tf.float32)  # (4, 3)\n",
    "\n",
    "temp_v = tf.constant([[   1,0],\n",
    "                      [  10,0],\n",
    "                      [ 100,5],\n",
    "                      [1000,6]], dtype=tf.float32)  # (4, 3)\n",
    "\n",
    "# This `query` aligns with the second `key`,\n",
    "# so the second `value` is returned.\n",
    "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
    "print_out(temp_q, temp_k, temp_v)\n",
    "print()\n",
    "\n",
    "# This query aligns with a repeated key (third and fourth), \n",
    "# so all associated values get averaged.\n",
    "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32)  # (1, 3)\n",
    "print_out(temp_q, temp_k, temp_v)\n",
    "print()\n",
    "\n",
    "# This query aligns equally with the first and second key, \n",
    "# so their values get averaged.\n",
    "temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
    "print_out(temp_q, temp_k, temp_v)\n",
    "print()\n",
    "\n",
    "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
    "print_out(temp_q, temp_k, temp_v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-head attention\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "  def __init__(self, d_model, num_heads):\n",
    "    super(MultiHeadAttention, self).__init__()\n",
    "    self.num_heads = num_heads\n",
    "    self.d_model = d_model\n",
    "    \n",
    "    assert d_model % self.num_heads == 0\n",
    "    \n",
    "    self.depth = d_model // self.num_heads\n",
    "    \n",
    "    self.wq = tf.keras.layers.Dense(d_model)\n",
    "    self.wk = tf.keras.layers.Dense(d_model)\n",
    "    self.wv = tf.keras.layers.Dense(d_model)\n",
    "    \n",
    "    self.dense = tf.keras.layers.Dense(d_model)\n",
    "    \n",
    "  def split_heads(self, x, batch_size):\n",
    "    \"\"\"Split the last dimension into (num_heads, depth).\n",
    "    Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
    "    \"\"\"\n",
    "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "    return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "\n",
    "  def call(self, v, k, q, mask):\n",
    "    batch_size = tf.shape(q)[0]\n",
    "    \n",
    "    q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
    "    k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
    "    v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
    "    \n",
    "    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
    "    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
    "    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
    "    \n",
    "    # scaled_attention.shape == (batch_size, num_heads, seq_len_v, depth)\n",
    "    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
    "    scaled_attention, attention_weights = scaled_dot_product_attention(\n",
    "        q, k, v, mask)\n",
    "    \n",
    "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_v, num_heads, depth)\n",
    "\n",
    "    concat_attention = tf.reshape(scaled_attention, \n",
    "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_v, d_model)\n",
    "\n",
    "    output = self.dense(concat_attention)  # (batch_size, seq_len_v, d_model)\n",
    "        \n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n",
    "y = tf.random.uniform((1, 60, 512))  # (batch_size, encoder_sequence, d_model)\n",
    "out, attn = temp_mha(y, k=y, q=y, mask=None)\n",
    "out.shape, attn.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feed forward network\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def point_wise_feed_forward_network(d_model, dff):\n",
    "  return tf.keras.Sequential([\n",
    "      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n",
    "      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n",
    "  ])\n",
    "\n",
    "\n",
    "class PointWiseFeedForwardNetwork(tf.keras.Model):\n",
    "    \n",
    "    def __init__(self, d_model, dff):\n",
    "        super(PointWiseFeedForwardNetwork, self).__init__(name='ffn')\n",
    "        self.d_model = d_model\n",
    "        self.dense_1 = tf.keras.layers.Dense(dff, activation='relu')\n",
    "        self.dense_2 = tf.keras.layers.Dense(d_model)\n",
    "    \n",
    "    def call(self, x):\n",
    "        x = self.dense_1(x)\n",
    "        return self.dense_2(x)\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        shapes = tf.shape(input_shape).as_list()\n",
    "        shapes[-1] = self.d_model\n",
    "        return tf.TensorShape(shapes)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 50, 512)\n",
      "\n",
      "(64, 50, 512)\n"
     ]
    }
   ],
   "source": [
    "sample_ffn = point_wise_feed_forward_network(512, 2048)\n",
    "print(sample_ffn(tf.random.uniform((64, 50, 512))).shape)\n",
    "print()\n",
    "\n",
    "ffn = PointWiseFeedForwardNetwork(512, 2048)\n",
    "print(ffn(tf.random.uniform((64, 50, 256))).shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder layer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LayerNormalization(tf.keras.layers.Layer):\n",
    "    def __init__(self, epsilon=1e-6, **kwargs):\n",
    "        self.eps = epsilon\n",
    "        super(LayerNormalization, self).__init__(**kwargs)\n",
    "    def build(self, input_shape):\n",
    "        self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n",
    "                                     initializer=tf.keras.initializers.Ones(), trainable=True)\n",
    "        self.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n",
    "                                    initializer=tf.keras.initializers.Zeros(), trainable=True)\n",
    "        super(LayerNormalization, self).build(input_shape)\n",
    "    def call(self, x):\n",
    "        mean = tf.keras.backend.mean(x, axis=-1, keepdims=True)\n",
    "        std = tf.keras.backend.std(x, axis=-1, keepdims=True)\n",
    "        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape\n",
    "\n",
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "    super(EncoderLayer, self).__init__()\n",
    "\n",
    "    self.mha = MultiHeadAttention(d_model, num_heads)\n",
    "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "    self.layernorm1 = LayerNormalization(epsilon=1e-6)\n",
    "    self.layernorm2 = LayerNormalization(epsilon=1e-6)\n",
    "    \n",
    "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "  def call(self, x, training, mask):\n",
    "\n",
    "    attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
    "    attn_output = self.dropout1(attn_output, training=training)\n",
    "    out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n",
    "    \n",
    "    ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
    "    ffn_output = self.dropout2(ffn_output, training=training)\n",
    "    out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
    "    \n",
    "    return out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([64, 43, 512])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n",
    "\n",
    "sample_encoder_layer_output = sample_encoder_layer(\n",
    "    tf.random.uniform((64, 43, 512)), False, None)\n",
    "\n",
    "sample_encoder_layer_output.shape  # (batch_size, input_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n",
    "               rate=0.1):\n",
    "    super(Encoder, self).__init__()\n",
    "\n",
    "    self.d_model = d_model\n",
    "    self.num_layers = num_layers\n",
    "    \n",
    "    self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
    "    self.pos_encoding = positional_encoding(input_vocab_size, self.d_model)\n",
    "    \n",
    "    \n",
    "    self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n",
    "                       for _ in range(num_layers)]\n",
    "  \n",
    "    self.dropout = tf.keras.layers.Dropout(rate)\n",
    "        \n",
    "  def call(self, x, training, mask):\n",
    "\n",
    "    seq_len = tf.shape(x)[1]\n",
    "    \n",
    "    # adding embedding and position encoding.\n",
    "    x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n",
    "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "    x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "    x = self.dropout(x, training=training)\n",
    "    \n",
    "    for i in range(self.num_layers):\n",
    "      x = self.enc_layers[i](x, training, mask)\n",
    "    \n",
    "    return x  # (batch_size, input_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 62, 512)\n"
     ]
    }
   ],
   "source": [
    "sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8, \n",
    "                         dff=2048, input_vocab_size=8500)\n",
    "\n",
    "sample_encoder_output = sample_encoder(tf.random.uniform((64, 62)), \n",
    "                                       training=False, mask=None)\n",
    "\n",
    "print (sample_encoder_output.shape)  # (batch_size, input_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder layer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "    super(DecoderLayer, self).__init__()\n",
    "\n",
    "    self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
    "    self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
    "\n",
    "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    " \n",
    "    self.layernorm1 = LayerNormalization(epsilon=1e-6)\n",
    "    self.layernorm2 = LayerNormalization(epsilon=1e-6)\n",
    "    self.layernorm3 = LayerNormalization(epsilon=1e-6)\n",
    "    \n",
    "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "    self.dropout3 = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    \n",
    "  def call(self, x, enc_output, training, \n",
    "           look_ahead_mask, padding_mask):\n",
    "    # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
    "\n",
    "    attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
    "    attn1 = self.dropout1(attn1, training=training)\n",
    "    out1 = self.layernorm1(attn1 + x)\n",
    "    \n",
    "    attn2, attn_weights_block2 = self.mha2(\n",
    "        enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
    "    attn2 = self.dropout2(attn2, training=training)\n",
    "    out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
    "    \n",
    "    ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
    "    ffn_output = self.dropout3(ffn_output, training=training)\n",
    "    out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
    "    \n",
    "    return out3, attn_weights_block1, attn_weights_block2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([64, 50, 512])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n",
    "\n",
    "sample_decoder_layer_output, _, _ = sample_decoder_layer(\n",
    "    tf.random.uniform((64, 50, 512)), sample_encoder_layer_output, \n",
    "    False, None, None)\n",
    "\n",
    "sample_decoder_layer_output.shape  # (batch_size, target_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "  def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, \n",
    "               rate=0.1):\n",
    "    super(Decoder, self).__init__()\n",
    "\n",
    "    self.d_model = d_model\n",
    "    self.num_layers = num_layers\n",
    "    \n",
    "    self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
    "    self.pos_encoding = positional_encoding(target_vocab_size, self.d_model)\n",
    "    \n",
    "    self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n",
    "                       for _ in range(num_layers)]\n",
    "    self.dropout = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "  def call(self, x, enc_output, training, \n",
    "           look_ahead_mask, padding_mask):\n",
    "\n",
    "    seq_len = tf.shape(x)[1]\n",
    "    attention_weights = {}\n",
    "    \n",
    "    x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
    "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "    x += self.pos_encoding[:, :seq_len, :]\n",
    "    \n",
    "    x = self.dropout(x, training=training)\n",
    "\n",
    "    for i in range(self.num_layers):\n",
    "      x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
    "                                             look_ahead_mask, padding_mask)\n",
    "      \n",
    "      attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
    "      attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
    "    \n",
    "    # x.shape == (batch_size, target_seq_len, d_model)\n",
    "    return x, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([64, 26, 512]), TensorShape([64, 8, 26, 62]))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8, \n",
    "                         dff=2048, target_vocab_size=8000)\n",
    "\n",
    "output, attn = sample_decoder(tf.random.uniform((64, 26)), \n",
    "                              enc_output=sample_encoder_output, \n",
    "                              training=False, look_ahead_mask=None, \n",
    "                              padding_mask=None)\n",
    "\n",
    "output.shape, attn['decoder_layer2_block2'].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(tf.keras.Model):\n",
    "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n",
    "               target_vocab_size, rate=0.1):\n",
    "    super(Transformer, self).__init__()\n",
    "\n",
    "    self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n",
    "                           input_vocab_size, rate)\n",
    "\n",
    "    self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n",
    "                           target_vocab_size, rate)\n",
    "\n",
    "    self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
    "    \n",
    "  def call(self, inp, tar, training, enc_padding_mask, \n",
    "           look_ahead_mask, dec_padding_mask):\n",
    "\n",
    "    enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n",
    "    \n",
    "    # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
    "    dec_output, attention_weights = self.decoder(\n",
    "        tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
    "    \n",
    "    final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
    "    \n",
    "    return final_output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([64, 26, 8000])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_transformer = Transformer(\n",
    "    num_layers=2, d_model=512, num_heads=8, dff=2048, \n",
    "    input_vocab_size=8500, target_vocab_size=8000)\n",
    "\n",
    "temp_input = tf.random.uniform((64, 62))\n",
    "temp_target = tf.random.uniform((64, 26))\n",
    "\n",
    "fn_out, _ = sample_transformer(temp_input, temp_target, training=False, \n",
    "                               enc_padding_mask=None, \n",
    "                               look_ahead_mask=None,\n",
    "                               dec_padding_mask=None)\n",
    "\n",
    "fn_out.shape  # (batch_size, tar_seq_len, target_vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HParams\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_layers = 4\n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "\n",
    "input_vocab_size = 100\n",
    "target_vocab_size = 100\n",
    "dropout_rate = 0.1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "  def __init__(self, d_model, warmup_steps=4000):\n",
    "    super(CustomSchedule, self).__init__()\n",
    "    \n",
    "    self.d_model = d_model\n",
    "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "\n",
    "    self.warmup_steps = warmup_steps\n",
    "    \n",
    "  def __call__(self, step):\n",
    "    arg1 = tf.math.rsqrt(step)\n",
    "    arg2 = step * (self.warmup_steps ** -1.5)\n",
    "    \n",
    "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Train Step')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VeW1+P/PSkISkpBAJghjmCUoThEtzhOgtxV71Yq1/XJbLbetdtDvrUPv99pe7/X31Q5qvWp7uWqLVotUa4v9OlfrVBkCKjIUyDkghCknAQJJIJBk/f7YT+AQT5KT5Oyck2S9X6+8ss/ez372OieQlb2fZ68tqooxxhgTa0nxDsAYY0zfZAnGGGOMLyzBGGOM8YUlGGOMMb6wBGOMMcYXlmCMMcb4whKMMcYYX1iCMcYY4wtLMMYYY3yREu8A4ik/P1+Li4vjHYYxxvQqK1eurFLVgo7a9esEU1xcTFlZWbzDMMaYXkVEPo2mnV0iM8YY4wtLMMYYY3xhCcYYY4wvLMEYY4zxha8JRkRmi8gGESkXkTsibE8TkWfd9mUiUhy27U63foOIzApb/4SIVIrImjaO+b9FREUk34/3ZIwxJjq+JRgRSQYeAS4DSoDrRKSkVbMbgL2qOgF4ALjP7VsCzAWmArOBR11/AL9x6yIdcxQwE9ga0zdjjDGm0/w8g5kOlKtqUFUPA4uAOa3azAEWuuXngItFRNz6RaraoKqbgXLXH6r6DrCnjWM+ANwG2GM6jTEmzvxMMCOAbWGvK9y6iG1UtRGoAfKi3Pc4IjIH2K6qH3cv7MSlqixesY3ahsZ4h2KMMR3qE4P8IpIB/BC4K4q280WkTETKQqGQ/8HF0Efb9nHb86u5/bnV8Q7FGGM65GeC2Q6MCns90q2L2EZEUoAcoDrKfcONB8YCH4vIFtd+lYgMa91QVReoaqmqlhYUdFjpIKFs3VMPwOvrdsc5EmOM6ZifCWYFMFFExopIKt6g/ZJWbZYA89zy1cCbqqpu/Vw3y2wsMBFY3taBVPUTVS1U1WJVLca7pHaaqu6K7VuKr0CoDoDDTc1sra6PczTGGNM+3xKMG1O5GXgVWA8sVtW1InK3iFzhmj0O5IlIOXArcIfbdy2wGFgHvALcpKpNACLyO+ADYLKIVIjIDX69h0QTCNUi4i2/snZnfIMxxpgOiHfC0D+VlpZqbyp2edkv3mVodhqhAw2kpiTxwrfPjndIxph+SERWqmppR+36xCB/f9DcrGyuqmV8QRaXn1TEh1v3sbPmYLzDMsaYNlmC6SV21Bzk0JFmxhVkMvtEb+7CK2v61BCTMaaPsQTTSwTdAP/4gizGF2QxeeggXvx4R5yjMsaYtlmC6SUCoVoAxhVkAjDn1OGs2rqPT6vr4hmWMca0yRJMLxEM1TEoPYWCrDQArjzFK2zwxw/tLMYYk5gswfQSgVAt4wqyEDdPefjggZw1LpcXPqygP88ENMYkLkswvUQwVMf4/Mzj1v3jqSPZUl3Ph9v2xSkqY4xpmyWYXqC2oZFd+w8xvjDruPWXnTSMtJQkXljVXhUdY4yJD0swvcBmN4NsXKszmEHpA7i0ZCgvrt5BQ2NTPEIzxpg2WYLpBYJV3gyy1mcwANeUjmJf/RFeW2sFMI0xicUSTC8QqKwlSWBMXsZntp07IZ+RQwbyzDJ7iKcxJrFYgukFAlV1jBySQVpK8me2JSUJ100fzQfBaoLuXhljjEkElmB6gUBlLeMLMtvcfk3pSFKShEUrtrXZxhhjepolmATX3Kxsqa5jXMFnx19aFA5K55IpQ3luZYUN9htjEoYlmATXUuRyfDsJBuDLZ45mT91hK4BpjEkYlmASXMtTLMe1c4kM4JwJ+YzNz+SJ97fYnf3GmIRgCSbBtQzcd3QGk5QkfO3sYj7eto9VW/f2RGjGGNMuSzAJLhCqZVB6CvlZqR22veq0kWSnp/D4e5t7IDJjjGmfJZgEFwzVHVfksj2ZaSlcd+ZoXlmzi2176nsgOmOMaZslmAQXDNW1O0W5tX+aUUySCL/52xb/gjLGmCj4mmBEZLaIbBCRchG5I8L2NBF51m1fJiLFYdvudOs3iMissPVPiEiliKxp1ddPReTvIrJaRF4QkcF+vreecLTIZQfjL+GKcgZy+UlFPLtiG/vqD/sYnTHGtM+3BCMiycAjwGVACXCdiJS0anYDsFdVJwAPAPe5fUuAucBUYDbwqOsP4DduXWuvAyeq6jRgI3BnTN9QHGw++pjk6M9gAL51wXhqGxrtLMYYE1d+nsFMB8pVNaiqh4FFwJxWbeYAC93yc8DF4g02zAEWqWqDqm4Gyl1/qOo7wJ7WB1PV11S10b1cCoyM9Rvqaccekxz9GQzAlKJsLpkylCfe28yBQ0f8CM0YYzrkZ4IZAYTXLqlw6yK2ccmhBsiLct/2fB14OdIGEZkvImUiUhYKhTrRZc8LhtouctmR7148gf2HGnlq6ac+RGaMMR3rc4P8IvKvQCPwdKTtqrpAVUtVtbSgoKBng+ukQKiOUbmRi1x2ZNrIwZw/qYDH3t1M/eHGjncwxpgY8zPBbAdGhb0e6dZFbCMiKUAOUB3lvp8hIv8EfB64XvvA7eyBUO1nHjLWGd+5aAJ76g7zWzuLMcbEgZ8JZgUwUUTGikgq3qD9klZtlgDz3PLVwJsuMSwB5rpZZmOBicDy9g4mIrOB24ArVLXX3wTS3Kxsrqrr1Ayy1kqLczl3Yj6//GuA/TYWY4zpYb4lGDemcjPwKrAeWKyqa0XkbhG5wjV7HMgTkXLgVuAOt+9aYDGwDngFuElVmwBE5HfAB8BkEakQkRtcXw8Dg4DXReQjEfmVX++tJ2zfd5CGxuZOD/C3dvvsE9hbf4T/eScYo8iMMSY6KX52rqovAS+1WndX2PIh4Jo29r0HuCfC+uvaaD+hW8EmmGBV16Yot3biiBw+P62Ix97dzFc/N4bCQemxCM8YYzrU5wb5+4pAZdemKEfyLzMnc6Spmf/6S3m3+zLGmGhZgklQwaroi1x2pDg/k2vPGMXvlm9lszszMsYYv1mCSVBeDbLoilxG43uXTCQtJYl7/t+6mPRnjDEdsQSToAKh2g4fMtYZhYPS+e7FE3ljfSVvbaiMWb/GGNMWSzAJqLahkd37G7o1RTmSr509lnH5mfzHi+s43Ngc076NMaY1SzAJ6NhTLGN3BgOQmpLEv32hhGBVHb/5mz2UzBjjL0swCSjoqijHYgZZaxdOLuSiEwr5xRub2FlzMOb9G2NMC0swCSjQjSKX0fjRF0poUuXf/riWPlBRxxiToCzBJKBgN4pcRmNMXia3XDKJN9bv5uU1u3w5hjHGWIJJQIFQbcwH+Fu74ZyxTB2ezV1/WktNvdUpM8bEniWYBNNS5LI7VZSjkZKcxH1XTWNv/WH+v5fW+3osY0z/ZAkmwbQUuRxf6O8ZDHh1ym48dyzPlm3jrb/bvTHGmNiyBJNgjj4m2eczmBa3XDKJE4YN4gfPraa6tqFHjmmM6R8swSQYP6coR5I+IJkHrj2F/QePcOcfPrFZZcaYmLEEk2CCVbVkx6jIZbSmFGXzg1mTeW3dbn5fVtFjxzXG9G2WYBJMoLKOcTEschmtG84Zy+fG5fHvL649epnOGGO6wxJMgglW+T9FOZKkJOH+a08mbUAy3/7tKg4eburxGIwxfYslmARy4NARdu9viGkV5c4oyhnIg9eewsbKA/yfP66x8RhjTLdYgkkgm2P0mOTuOG9SAd+5aCLPr6pgcdm2uMVhjOn9fE0wIjJbRDaISLmI3BFhe5qIPOu2LxOR4rBtd7r1G0RkVtj6J0SkUkTWtOorV0ReF5FN7vsQP9+bHwJHqyj3/CWycN+7eCLnTMjn3/60ljXba+IaizGm9/ItwYhIMvAIcBlQAlwnIiWtmt0A7FXVCcADwH1u3xJgLjAVmA086voD+I1b19odwF9UdSLwF/e6VwmG6kgSGO1TkctoJScJD849hfzMVL7xZBmV+w/FNR5jTO/k5xnMdKBcVYOqehhYBMxp1WYOsNAtPwdcLN70qTnAIlVtUNXNQLnrD1V9B9gT4XjhfS0Erozlm+kJwVAdo30sctkZ+Vlp/M+8UvbVH2H+Uys5dMQG/Y0xneNnghkBhF/Er3DrIrZR1UagBsiLct/WhqrqTre8CxgaqZGIzBeRMhEpC4VC0byPHuM9Jjm+l8fCTR2ewwPXnsxH2/Zxx/OrbdDfGNMpfXKQX73fhBF/G6rqAlUtVdXSgoKCHo6sbU2uyGU8B/gjmX1iEf8ycxJ//GgH//VmebzDMcb0In4mmO3AqLDXI926iG1EJAXIAaqj3Le13SJS5PoqAnpV9cYdrshlIp3BtLjpwgn842kjuP/1jSxavjXe4Rhjegk/E8wKYKKIjBWRVLxB+yWt2iwB5rnlq4E33dnHEmCum2U2FpgILO/geOF9zQP+FIP30GN6ushlZ4gI9101jfMnFfDDFz7h9XW74x2SMaYX8C3BuDGVm4FXgfXAYlVdKyJ3i8gVrtnjQJ6IlAO34mZ+qepaYDGwDngFuElVmwBE5HfAB8BkEakQkRtcX/cCl4rIJuAS97rXaCly2RNl+rtiQHISj15/GieNyOHmZ1axYkukeRbGGHOM9OeB29LSUi0rK4t3GAD86wuf8OLHO/j4RzN7vA5ZZ1TXNnDNrz6gqraBZ75xFieOyIl3SMaYHiYiK1W1tKN2fXKQvzcKhuoYX9jzRS47Ky8rjYVfn05WWgpfeXwZ63fuj3dIxpgEZQkmQQRCtYzLT8zLY62Nys3gd/PPIj0lmesfW8aGXQfiHZIxJgFZgkkABw4dofJA/IpcdsWYvEx+N/8sUpKE6x9bSnmlJRljzPGiSjAico6IfM0tF7iZXSZGjg7wJ+AU5faMzfeSDAhzFyxl3Q67XGaMOabDBCMiPwJuB+50qwYAv/UzqP4mWNVS5LL3nMG0GF+QxbP/fBapyUlcu+ADymx2mTHGieYM5ovAFUAdgKruAAb5GVR/EwzVkZwkcS9y2VXjC7L4/bdmkJ+VxlceX8bbGxOrBI8xJj6iSTCHw0uviEjv+zM7wQVCtYwaMjAhilx21YjBA1n8z59jbH4WNy5cwZ9X74h3SMaYOIsmwSwWkf8GBovIN4A3gMf8Dat/CYbqet34SyQFg9JYNP8sTh45mJuf+ZBfvR2wApnG9GMdJhhV/RleKf3ngcnAXar6kN+B9RdNzUqwqq5XzSBrT87AAfz2xjP5/LQi7n357/zwhU840tQc77CMMXGQ0lEDEblPVW8HXo+wznTTjn0HOZygRS67Kn1AMg/NPZUxeRk88laAir0HeeT608hOHxDv0IwxPSiaS2SXRlh3WawD6a8S5THJsZaUJPxg1gn85KppfBCo5qpH/0bQvVdjTP/QZoIRkW+JyCd4RSVXh31tBlb3XIh9W8DdA9NXLpG19qUzRvHk16dTVdvAnIfft0rMxvQj7Z3BPAN8Aa8M/hfCvk5X1a/0QGz9QjBUS87AAeRlpsY7FN/MmJDPi985h+L8TL7xZBn3v76R5mYb/Demr2szwahqjapuUdXrVPVT4CDeVOUsERndYxH2cd5jkjMTvshld40cksHvv/k5rj59JA/9ZRM3LFzB3rrD8Q7LGOOjaO7k/4J7xspm4G1gC/Cyz3H1G8FQXa8pctld6QOS+enV0/iPK0/k/fJqLvvFuywNVsc7LGOMT6IZ5P9P4Cxgo6qOBS4GlvoaVT/RUuRyfGHfHH+JRET46llj+MO3ZzAwNZnr/mcp97+2gUabymxMnxNNgjmiqtVAkogkqepbQIcPmjEdayly2V/OYMKdOCKHP3/nHK46bSQPvVnOtQuWUrG3Pt5hGWNiKJoEs09EsoB3gKdF5Be4umSme1qKXE7oR2cw4TLTUvjZNSfzi7mnsGHXAWY/+C7Prthqd/8b00dEk2DmAPXALcArQABvNpnppkClK3KZ2z8TTIs5p4zg5e+dy4kjsrn9+U/4p1+vYGfNwXiHZYzppmhKxdSparOqNqrqQuBhYHY0nYvIbBHZICLlInJHhO1pIvKs275MRIrDtt3p1m8QkVkd9SkiF4vIKhH5SETeE5EJ0cQYT8GqWkbnZpCaYs99G5WbwTM3nsW/XzGV5Zv3MPOBd1hcts3OZozpxdq70TLb/ZJ/WERmiudmIAh8qaOORSQZeATvrv8S4DoRKWnV7AZgr6pOAB4A7nP7lgBzgal4yexREUnuoM9fAter6il49/D8n+g+gvgJVNYxLr9/n72ES0oS5s0o5pXvn8uUYdnc9txq/tcTy9lSZVdkjemN2vvT+Sm84pafADcCbwHXAFeq6pwo+p4OlKtqUFUPA4vwLreFmwMsdMvPAReLd0PIHGCRqjao6mag3PXXXp8KZLvlHCCh68U3NSubq/tOkctYGpOXyaL5Z3H3nKl8tHUfMx98h1+8sYmGxqZ4h2aM6YT2il2OU9WTAETkMWAnMFpVD0XZ9whgW9jrCuDMttqoaqOI1AB5bv3SVvuOcMtt9Xkj8JKIHAT2402tTlgtRS77Wg2yWElKEv7X54qZNXUY//HndTzwxkb+9NF2/vPKE5kxIT/e4RljotDeGcyRlgVVbQIqOpFc4uEW4HJVHQn8Grg/UiMRmS8iZSJSFgrF78mL5a7wY1+qouyHodnpPPzl01j49ek0qfLlx5Zx8zOrbEqzMb1AewnmZBHZ774OANNalkVkfxR9bwdGhb0e6dZFbCMiKXiXtqrb2TfiehEpAE5W1WVu/bPAjEhBqeoCVS1V1dKCgoIo3oY/Wu6BGW+XyKJy/qQCXv3+eXzv4om8sX43F/38bX766t+pbWiMd2jGmDa0V4ssWVWz3dcgVU0JW85ua78wK4CJIjJWRFLxBu2XtGqzBJjnlq8G3nSPZ14CzHWzzMYCE4Hl7fS5F8gRkUmur0uB9dF8APEScEUuc/twkctYSx+QzC2XTuLN/30Bl584jEfeCnDhz/7K4hXbaLLimcYkHN/mx6pqI3Az8CreL/vFqrpWRO4WkStcs8eBPBEpB24F7nD7rgUWA+vw7r25SVWb2urTrf8G8LyIfAx8FfiBX+8tFoL9pMilH4YPHsiDc0/lhW/PYNSQgdz2/Go+/1/v8ebfd9u0ZmMSiPTn/5ClpaVaVlYWl2Ofcc8bnD+pgJ9dc3Jcjt9XqCovrt7Jz1/bwKfV9Zw+Zgg/mDWZs8blxTs0Y/osEVmpqh2WDLM7/OLgwKEjhA402BTlGBARrjh5OG/cej73fPFEKvbWM3fBUr76+DJWV+yLd3jG9GuWYOLg2AC/zSCLlQHJSVx/5hje/sGF/PDyE/hkew1XPPw+Ny5cwYdb98Y7PGP6pWieB3MgbDZZy9c2EXlBRMb1RJB9TcBNUbYZZLGXPiCZ+eeN593bLuSWSyaxYstevvjo3/jKY8tYGqy2MRpjelB7N1q2eBDvhsZnAMGbuTUeWAU8AVzgV3B9VTBkRS79Nih9AN+7ZCI3nDuW3y79lMfeDTJ3wVJKxwzhposmcMGkAptgYYzPorlEdoWq/reqHlDV/aq6AJilqs8CQ3yOr08KhKzIZU/JSkvhm+eP573bL+Lfr5jK9n0H+dqvV3D5Q+/x3MoKKz9jjI+i+Q1XLyJfEpEk9/UloOWOfrve0AXeY5Lt7KUnpQ9IZt6MYt7+wYX85KppNDU38y+//5iz732Lh/6yierahniHaEyfE02CuR7vvpJKYLdb/oqIDMS7J8V0QkuRy/GFNsAfD6kpSXzpjFG8+v3zePLr05k6PJv7X9/IjHvf5I7nV7Nx94F4h2hMn9HhGIyqBmn7AWPvxTacvm/7Xq/IpZ3BxJeIcN6kAs6bVEB55QEef28Lf1hVwaIV2/jcuDyuP2s0M0uG2WVMY7qhwwTj6nx9AygOb6+qX/cvrL4r4B6TbGcwiWNC4SD+7z+exA9mTeZ3y7fyzLKt3PzMh+RnpfKl0lFcN300o3Iz4h2mMb1ONLPI/gS8C7wB2IhoNwUqXRVlO4NJOLmZqdx04QS+ef543tkU4umlW/nV2wF++XaA8yYWcP2Zo7nohEJSku2sxphoRJNgMlT1dt8j6SeCVXUMzrAil4ksOUm4cHIhF04uZMe+gzy7YhuLVmxl/lMryc9K5cpTRnDV6SOZUhRNzVdj+q9oEsyfReRyVX3J92j6gUBlLePyrchlbzF88EBuuXQS37loAm9tCPHcym0s/GALj723mZKibK46fSRzThlOflZavEM1JuF0WOzSPQsmE2jAewiZABplyf6EFo9il1bksvfbU3eYFz/ewfOrKlhdUUNKknDB5AK+eOpILjqhkIGpyfEO0RhfRVvsMppZZINiE5LZ74pcWg2y3i03M5V5M4qZN6OYjbsP8PyqCv744XbeWF9JRmoyl0wZyuenFXH+5ALSUizZmP6rzQQjIieo6t9F5LRI21V1lX9h9U0tRS6tinLfMWnoIO68bAq3zTqBZcFqXly9k5fX7GTJxzsYlJ7CzJJhfP7kIs6ZkM8Amxxg+pn2zmBuBeYDP4+wTYGLfImoDwseLXJpZzB9TXKSMGNCPjMm5HP3nKm8X17Fn1fv5NW1u3h+VQWDMwYws2QoM0uGcc7EfNIH2JmN6fvaTDCqOt99v7DnwunbAqFaV+TS7qnoywYkJ3HB5EIumFzIPV88kXc2VvHn1Tt4+ZNdLC6rICM1mfMnFTBz6lAumjyUnIwB8Q7ZGF9EM4sMEZnBZ2+0fNKnmPqsYKjOilz2M2kpyVxaMpRLS4ZyuLGZpcFqXlu3i9fW7ublNbtISRLOHJfLzJJhXFIylBGDB8Y7ZGNiJppZZE/hlef/iGM3Wqqqftfn2HzX07PIZj7wNqNzM3hs3hk9dkyTmJqblY8r9vHaut28tnYXATc+N2loFhdOLuT8yQWUjsm1P0ZMQorZLDKgFCjRLjypSURmA78AkoHHVPXeVtvTgCeB04Fq4FpV3eK23QncgJfUvquqr7bXp3g3lvwncI3b55eq+lBnY/ZLU7OypbqeCyYXxjsUkwCSkoRTRw/h1NFDuH32CZRX1vLXDZW8taGSJ97fzH+/EyQrLYVzJuRzweQCLphcyLCc9HiHbUynRJNg1gDDgJ2d6VhEkoFHgEvxHli2QkSWqOq6sGY3AHtVdYKIzAXuA64VkRK8B5tNBYYDb4jIJLdPW33+EzAKOEFVm0UkoX6TtxS5tKdYmkgmFGYxoTCLG88dR21DI38rr+KtDSHe3lDJK2t3ATClKJvzJuZz9oR8zijOtfttTMKLJsHkA+tEZDnezZYAqOoVHew3HSh31ZgRkUXAHCA8wcwBfuyWnwMedmcic4BFqtoAbBaRctcf7fT5LeDLqtrs4quM4r31mJbHJI+zGWSmA1lpKcycOoyZU4ehqmzcXctbGyr5a9jZTWpyEqeNGczZ4/M5e2I+00bkWI00k3CiSTA/7mLfI4BtYa8rgDPbaqOqjSJSA+S59Utb7TvCLbfV53i8s58vAiG8y2qbuhh7zAVsirLpAhFh8rBBTB42iG+eP576w42s2LKXv5VX8V55Ffe/sZGfv76RQWkpnDkuj7Mn5HH2hHwmFGSRlGTliEx8tZtg3GWuH/eSqcppwCFVLRWRfwSeAM5t3UhE5uPd38Po0aN7LLhAyIpcmu7LSE3h/EkFnD+pAPDK1nwQqOb9QBXvl1fxxvrdAAzJGMAZxblMH5vLmWPzmFI0yM5wTI9rN8GoapOINItIjqrWdLLv7XhjIi1GunWR2lSISAqQgzfY396+ba2vAP7gll8Afh0pKFVdACwAbxZZ9G+ne4KhWivRb2IuNzOVf5hWxD9MKwJg2556PghWs2LzHpZv2cNr67yEk5mazOnFuZw51ks600bmWBkb47toLpHVAp+IyOtAXcvKKKYprwAmishYvCQwF/hyqzZLgHnAB8DVwJuqqiKyBHhGRO7HG+SfCCzHK7TZVp9/BC4ENgPnAxujeG89JlhVxwXur05j/DIqN4NRuRl8qdT7O2z3/kMs37zn6NdPX90AeI+OPmXUYErHDHGz2QZbRWgTc9EkmD9w7Mwgam5M5WbgVbwpxU+o6loRuRsoU9UlwOPAU24Qfw9ewsC1W4w3eN8I3KSqTQCR+nSHvBd4WkRuwUuKN3Y2Zr+0FLm0AX7T04Zmp/OFk4fzhZOHA7C37jArtnjJZsWWPSx4J0hjs3ciPzo3g1NHD+Y0l3CmFGVb/TTTLR3eaNmX9dSNlh9t28eVj7zPgq+ezsypw3w/njHROnSkiTXba1i1dS8fbt3Hqq172b3fmyyalpLEtJE53hnOqMGcPGowRTnp9iwjE7sbLUVkIvB/gRLg6J1eqjquWxH2I0cfk2xnMCbBpA9IprQ4l9Li3KPrduw7yIdb9/Hh1r2s2rqX37y/hQVNzQDkZ6Vy4ogcTmr5GpnDsGxLOiayaC6R/Rr4EfAA3hjH1wA7b+6EYJUVuTS9x/DBAxk+eODRiQMNjU2s33mATyr2sbqihk+21/Dupiqa3KW1/Kw0ThqRzUkjB3PSiBymjcxhaLZVHTDRJZiBqvoXERFV/RT4sYisBO7yObY+I1BZxxgrcml6qbSUZE4ZNZhTRg0+uu7g4SbW79rPJxU1rK6oYc32Gt7euAmXcygYlMaUomymFA2ipCibKUXZjMvPtKnS/Uw0CaZBRJKATW6AfTtg13o6IVhVaw8ZM33KwNRkThs9hNNGDzm6rv5wI+t37ncJZz/rd+7niUAVR5q8rJOaksSkoVlMGZbtkk82JUXZ9riCPiyaBPM9IAP4LvAfeJfJ5vkZVF/S1KxsqarnQityafq4jNQUTh+Ty+ljjo3nHG5sJhCqZf3O/e7rAG/+vZLfr6w42mZ4TjpTirI5oWgQk4YOYmLhIMYXZtp9On1AhwlGVVcAiEizqn7N/5D6loq99RxuarYzGNMvpaYkHT1baaGqhA40sM4lnJbk89eNoaPjOkkCxXmZTBya5SWdoYOYNDSLcflZdqm5F4lmFtnn8O5XyQJGi8jJwD+r6rf9Dq4vCLrnfFgNMmM8IkJhdjqF2enHPb6iobGJzVV1bNxdy6Zm5kYNAAATvUlEQVTdB9i4+wCbdtfy+rrdR8d2kpOE4ryMzySdsfmZVl06AUVziexBYBbeXfeo6scicp6vUfUhVkXZmOikpSRzwrBsThiWfdz6Q0eaCIbq2FTpJZ2Nu71Lbq+s3UX4bXwjBg9kXEEm4/IzGVeQxfiCLMYVZDIsO90Kf8ZJVI9MVtVtrea5N7XV1hzPilwa0z3pA5IpGZ5NyfDIiSdYVet9D9USCNXx3MoK6g4f+xU1cEAyY/MzveRTkMX4gkzvrKcgk6y0qH4Fmi6K5tPdJiIzABWRAXiD/uv9DavvCIZq7fKYMT5oK/GoKpUHGgiEWhKPl4RWV9Tw0ic7j15uA+8enuK8DEbnZTAmN5Pi/AxG52ZQnJfJ4IwBdgNpN0WTYL6J94jiEXhTlF8DbPwlSoFQHRdOtiKXxvQUEWFodjpDs9OZMT7/uG2HjjSxdU/90bOdrdX1bKmu44NANX9YdXyx90HpKYzJy2BMXiZjXNIZnZfBmLwMhg6yy27RiGYWWRVwffg6Efk+3tiMaUfNwSNU1TYwvtDOYIxJBOkDkpk01JsO3dqhI01s21PPpy7pbN1Tz5bqetZur+HVNbuOFgUFr07b6Fwv2YwcksHIIQPdl7ecM9DOfiDKMZgIbsUSTIeCLQP89hwYYxJe+oBkJrrZaa01NjWzY98hPt1Tx5bqerZWe9+37alnaXAPtQ2Nx7UflJbCiLCEE558Rg3JIHtgSr9IQF1NMH3/k4mBlinKNoPMmN4tJTmJ0W6s5tyJx29TVWoOHqFi70Eq9ta77y3L9SwNVneYgEYMHkjR4HSKcgYyfHA6hYPSSe4Dl+C6mmD6b43/TgiEaklJEsbkWZFLY/oqEWFwRiqDM7xK0611JQElJwlDB6VRNHggRTnpXgHSnHSKBg9keI6XjPIyUxP+LKjNBCMiB4icSAQY6FtEfUgwVMfo3Ax7aJMx/Vg0CWj/oUZ21hxk575D7Gj1fc32Gl5bt5vDjc3H7ZeakkRRTrqXgHKOnQENcxMchuakkZ+ZFtfJCG0mGFX97IVI0ylekUu7PGaMaZuIkDNwADkDB3zmJtMWqsqeusPsrDnEjn0Hve8uAe2sOciyzXvYtf/Q0VI7LVKShMJBaQzNST+aeIa55Rnj8yj0+bEKdpeRT6zIpTEmVkSEvKw08rLSIp4Fgfc7p6q2gV01h9i1/xC79x86bnnj7gO8u6nq6OW4J78+3RJMb9VS5NJusjTG9ITkpGP3/5zcTrvahkZ21RyiKMf/h8JZgvHJsRpkNkXZGJM4stJSmNBD9+b5OvosIrNFZIOIlIvIHRG2p4nIs277MhEpDtt2p1u/QURmdaLPh0Sk1q/3FC2bomyM6e98SzAikgw8AlwGlADXiUhJq2Y3AHtVdQLwAHCf27cEmAtMBWYDj4pIckd9ikgpMIQEEAjVMcSKXBpj+jE/z2CmA+WqGlTVw8AiYE6rNnOAhW75OeBi8SZ2zwEWqWqDqm4Gyl1/bfbpks9Pgdt8fE9RC4RsBpkxpn/zM8GMALaFva5w6yK2UdVGoAbIa2ff9vq8GViiqjvbC0pE5otImYiUhUKhTr2hzgiG6hhv4y/GmH6sT9wBKCLDgWuA/+qoraouUNVSVS0tKPCnynFLkUs7gzHG9Gd+JpjtwKiw1yPduohtRCQFyAGq29m3rfWnAhOAchHZAmSISHms3khnWZFLY4zxN8GsACaKyFgRScUbtF/Sqs0SYJ5bvhp4U1XVrZ/rZpmNBSYCy9vqU1X/n6oOU9ViVS0G6t3EgbgIuBlkVqbfGNOf+XYfjKo2isjNwKtAMvCEqq4VkbuBMlVdAjwOPOXONvbgJQxcu8XAOqARuElVmwAi9enXe+iqoCtyOTrXilwaY/ovX2+0VNWXgJdarbsrbPkQ3thJpH3vAe6Jps8IbeJ66hAM1TE6z4pcGmP6N/sN6INAqJZx+XZ5zBjTv1mCibHGpmY+ra5nfKEN8Btj+jdLMDFWsfegV+TSzmCMMf2cJZgYC1ZZkUtjjAFLMDHXUuTSyvQbY/o7SzAxFgjVMiRjAEOsyKUxpp+zBBNjgVCdnb0YYwyWYGIuGKq18RdjjMESTEzV1B+hqvawFbk0xhgswcRUwM0gs0tkxhhjCSamjj0m2S6RGWOMJZgYsiKXxhhzjCWYGAqEaq3IpTHGOPabMIaCNkXZGGOOsgQTI41NzWyprrPxF2OMcSzBxEjF3oMcaVIrcmmMMY4lmBhpKXJpZfqNMcZjCSZGApVuirKdwRhjDGAJJmaCVbXkZqZakUtjjHF8TTAiMltENohIuYjcEWF7mog867YvE5HisG13uvUbRGRWR32KyNNu/RoReUJEBvj53loLVNYxLt8ujxljTAvfEoyIJAOPAJcBJcB1IlLSqtkNwF5VnQA8ANzn9i0B5gJTgdnAoyKS3EGfTwMnACcBA4Eb/XpvkQSrrMilMcaE8/MMZjpQrqpBVT0MLALmtGozB1jolp8DLhYRcesXqWqDqm4Gyl1/bfapqi+pAywHRvr43o7TUuTS7oExxphj/EwwI4BtYa8r3LqIbVS1EagB8trZt8M+3aWxrwKvdPsdRClw9DHJlmCMMaZFXxzkfxR4R1XfjbRRROaLSJmIlIVCoZgc8Nhjku0SmTHGtPAzwWwHRoW9HunWRWwjIilADlDdzr7t9ikiPwIKgFvbCkpVF6hqqaqWFhQUdPItRRZwRS5HWZFLY4w5ys8EswKYKCJjRSQVb9B+Sas2S4B5bvlq4E03hrIEmOtmmY0FJuKNq7TZp4jcCMwCrlPVZh/f12cEQ7WMsSKXxhhznBS/OlbVRhG5GXgVSAaeUNW1InI3UKaqS4DHgadEpBzYg5cwcO0WA+uARuAmVW0CiNSnO+SvgE+BD7x5AvxBVe/26/2FC4TqbPzFGGNa8S3BgDezC3ip1bq7wpYPAde0se89wD3R9OnW+/pe2tLY1Myn1XVcPKUwHoc3xpiEZdd0uulokUs7gzHGmONYgummQMgVubQZZMYYcxxLMN3UMkXZilwaY8zxLMF0UyBkRS6NMSYSSzDdFAxZkUtjjInEEkw3BUK1NsBvjDERWILphpr6I1TXHbYqysYYE4ElmG5oKXJpZzDGGPNZlmC6IVDZUkXZzmCMMaY1SzDdEKyqY0CyFbk0xphILMF0Q6CyltG5VuTSGGMisd+M3RCssiKXxhjTFkswXdRS5NIG+I0xJjJLMF20zRW5tAF+Y4yJzBJMFwVDNkXZGGPaYwmmi6yKsjHGtM8STBcFQ3XkZaYyOMOKXBpjTCSWYLooEKq18RdjjGmHJZgu8qoo2/iLMca0xdcEIyKzRWSDiJSLyB0RtqeJyLNu+zIRKQ7bdqdbv0FEZnXUp4iMdX2Uuz59u3a1r/4w1XWHGV9oZzDGGNMW3xKMiCQDjwCXASXAdSJS0qrZDcBeVZ0APADc5/YtAeYCU4HZwKMiktxBn/cBD7i+9rq+fRGwp1gaY0yH/DyDmQ6Uq2pQVQ8Di4A5rdrMARa65eeAi0VE3PpFqtqgqpuBctdfxD7dPhe5PnB9XunXGzs6RbnQEowxxrTFzwQzAtgW9rrCrYvYRlUbgRogr51921qfB+xzfbR1rJgJhFyRyyED/TqEMcb0ev1ukF9E5otImYiUhUKhLvVRnJfBF08dQYoVuTTGmDb5+RtyOzAq7PVIty5iGxFJAXKA6nb2bWt9NTDY9dHWsQBQ1QWqWqqqpQUFBV14WzB3+mh+cvXJXdrXGGP6Cz8TzApgopvdlYo3aL+kVZslwDy3fDXwpqqqWz/XzTIbC0wElrfVp9vnLdcHrs8/+fjejDHGdCCl4yZdo6qNInIz8CqQDDyhqmtF5G6gTFWXAI8DT4lIObAHL2Hg2i0G1gGNwE2q2gQQqU93yNuBRSLyn8CHrm9jjDFxIt4f//1TaWmplpWVxTsMY4zpVURkpaqWdtTORqmNMcb4whKMMcYYX1iCMcYY4wtLMMYYY3xhCcYYY4wv+vUsMhEJAZ92cfd8oCqG4cSKxdU5FlfnWFydk6hxQfdiG6OqHd6p3q8TTHeISFk00/R6msXVORZX51hcnZOocUHPxGaXyIwxxvjCEowxxhhfWILpugXxDqANFlfnWFydY3F1TqLGBT0Qm43BGGOM8YWdwRhjjPGFJZguEJHZIrJBRMpF5I4eON4WEflERD4SkTK3LldEXheRTe77ELdeROQhF9tqETktrJ95rv0mEZnX1vE6iOUJEakUkTVh62IWi4ic7t5rudtXuhHXj0Vku/vcPhKRy8O23emOsUFEZoWtj/izdY+IWObWP+seF9FRTKNE5C0RWScia0Xke4nwebUTV1w/L7dfuogsF5GPXWz/3l5/4j3S41m3fpmIFHc15i7G9RsR2Rz2mZ3i1vfkv/1kEflQRP6cCJ/VcVTVvjrxhfeYgAAwDkgFPgZKfD7mFiC/1bqfAHe45TuA+9zy5cDLgABnAcvc+lwg6L4PcctDuhDLecBpwBo/YsF77s9Zbp+Xgcu6EdePgX+J0LbE/dzSgLHu55nc3s8WWAzMdcu/Ar4VRUxFwGlueRCw0R07rp9XO3HF9fNybQXIcssDgGXu/UXsD/g28Cu3PBd4tqsxdzGu3wBXR2jfk//2bwWeAf7c3mffU59V+JedwXTedKBcVYOqehhYBMyJQxxzgIVueSFwZdj6J9WzFO9Jn0XALOB1Vd2jqnuB14HZnT2oqr6D9+yemMfitmWr6lL1/uU/GdZXV+Jqyxxgkao2qOpmoBzv5xrxZ+v+krwIeC7Ce2wvpp2qusotHwDWAyOI8+fVTlxt6ZHPy8WjqlrrXg5wX9pOf+Gf5XPAxe74nYq5G3G1pUd+liIyEvgH4DH3ur3Pvkc+q3CWYDpvBLAt7HUF7f/njAUFXhORlSIy360bqqo73fIuYGgH8fkZd6xiGeGWYxnjze4SxRPiLkV1Ia48YJ+qNnY1Lnc54lS8v3wT5vNqFRckwOflLvl8BFTi/QIOtNPf0Rjc9hp3/Jj/P2gdl6q2fGb3uM/sARFJax1XlMfv6s/yQeA2oNm9bu+z77HPqoUlmN7hHFU9DbgMuElEzgvf6P7iSYjpgIkUC/BLYDxwCrAT+Hk8ghCRLOB54Puquj98Wzw/rwhxJcTnpapNqnoKMBLvr+gT4hFHa63jEpETgTvx4jsD77LX7T0Vj4h8HqhU1ZU9dczOsgTTeduBUWGvR7p1vlHV7e57JfAC3n+63e60Gve9soP4/Iw7VrFsd8sxiVFVd7tfCs3A/+B9bl2JqxrvEkdKq/UdEpEBeL/En1bVP7jVcf+8IsWVCJ9XOFXdB7wFfK6d/o7G4LbnuOP79v8gLK7Z7nKjqmoD8Gu6/pl15Wd5NnCFiGzBu3x1EfALEuiz8m1guq9+ASl4A3NjOTbwNdXH42UCg8KW/4Y3dvJTjh8o/olb/geOH1xc7tbnApvxBhaHuOXcLsZUzPGD6TGLhc8OdF7ejbiKwpZvwbvODDCV4wc1g3gDmm3+bIHfc/zA6bejiEfwrqU/2Gp9XD+vduKK6+fl2hYAg93yQOBd4PNt9QfcxPED14u7GnMX4yoK+0wfBO6N07/9Czg2yB/Xz+q4uLryC6a/f+HNENmId234X30+1jj3g/0YWNtyPLxrp38BNgFvhP0jFeARF9snQGlYX1/HG8ArB77WxXh+h3f55AjeNdkbYhkLUAqscfs8jLsZuItxPeWOuxpYwvG/QP/VHWMDYbN12vrZup/Dchfv74G0KGI6B+/y12rgI/d1ebw/r3biiuvn5fabBnzoYlgD3NVef0C6e13uto/rasxdjOtN95mtAX7LsZlmPfZv3+17AccSTFw/q/Avu5PfGGOML2wMxhhjjC8swRhjjPGFJRhjjDG+sARjjDHGF5ZgjDHG+MISjDGdJCJ5YdVzd8nxFYijrRr8axGZ3IljFonIS66a7zoRWeLWjxORuV19L8b4yaYpG9MNIvJjoFZVf9ZqveD9/2qOuGPnj/M4sEpVH3Gvp6nqahG5BLhZVaMqJmlMT7IzGGNiREQmuLOLp/Fuii0SkQUiUibeM0TuCmv7noicIiIpIrJPRO51ZycfiEhhhO6LCCuGqKqr3eK9wIXu7Om7rr/7xXt2yWoRudEd7xLxngHzsnu+xyMuCRrjG0swxsTWCcADqlqiXg25O1S1FDgZuFRESiLskwO8raonAx/g3end2sPAQhF5U0R+2FLLDK/UzFuqeoqqPgTMxyuAOB2vAONNIjLatT0T+Bbe8z+mEJ/HTJh+xBKMMbEVUNWysNfXicgqYBXeL/VICeagqr7sllfi1VQ7jqq+hFfp+HHXx4cikhehr5nA11xZ+WXAYGCi27ZUVbeoahNeccRzOvvmjOmMlI6bGGM6oa5lQUQmAt8DpqvqPhH5LV49qNYOhy030cb/S1WtBp4GnhaRV/ASRF2rZoJX3PAvx630xmpaD7jaAKzxlZ3BGOOfbOAAsD/saYZdIiIXi8hAt5yNV+F2q+t/UFjTV4Fvt5RrF5HJLfsBZ4nIaBFJBr4EvNfVeIyJhp3BGOOfVcA64O/Ap8D73ejrDOBhETmC94fhL1X1QzctOllEPsa7fPYIMBr4yI3hV3JsrGU5Xvn28XhVnJd0Ix5jOmTTlI3pB2w6s4kHu0RmjDHGF3YGY4wxxhd2BmOMMcYXlmCMMcb4whKMMcYYX1iCMcYY4wtLMMYYY3xhCcYYY4wv/n9c8/i9Wb3X+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learning_rate = CustomSchedule(d_model)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
    "                                     epsilon=1e-9)\n",
    "\n",
    "temp_learning_rate_schedule = CustomSchedule(d_model)\n",
    "\n",
    "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.xlabel(\"Train Step\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
